{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGBoost Parameter Tuning for Otto Dataset\n",
    "1. 直接调用xgboost内嵌的cv寻找最佳的参数n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = '~/Desktop/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "test = pd.read_csv(dpath +\"RentListingInquries_FE_test.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAELCAYAAAARNxsIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGx5JREFUeJzt3X2wHXWd5/H3h/AgPibIhcokYYKaGUUdo14hypSj6EDAHRMpUChHIkNtxAKF0rUEd5aMPMyM4ygrM8pupogkrkOIiBI1GDMMyII8JGB4CJHlioxcyZJgeNQVivDZP/p35eTm5N7OpU9OTu7nVdV1ur/96z7fwy340v379a9lm4iIiCbs0e0EIiJi95GiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIas2e3E9jZ9t9/f0+fPr3baURE9JTbbrvtEdt9o7Ubd0Vl+vTprFmzpttpRET0FEn/Uaddbn9FRERjOlZUJL1I0q2S7pC0TtLnS/xSSb+QtLYsM0tcki6SNCDpTklvaTnXPEn3lWVeS/ytku4qx1wkSZ36PRERMbpO3v56GjjC9lOS9gJukHR12fcZ21cMa380MKMshwEXA4dJ2g9YAPQDBm6TtNz2o6XNfOBmYAUwG7iaiIjoio5dqbjyVNncqywjzbM/B1hSjrsZmChpMnAUsMr25lJIVgGzy76X277J1fz9S4C5nfo9ERExuo72qUiaIGktsJGqMNxSdl1QbnFdKGmfEpsCPNhy+GCJjRQfbBNvl8d8SWskrdm0adML/l0REdFeR4uK7S22ZwJTgUMlvQE4G3gt8DZgP+CzpXm7/hCPId4uj4W2+2339/WNOiIuIiLGaKeM/rL9GHAdMNv2hnKL62ng68ChpdkgMK3lsKnAQ6PEp7aJR0REl3Ry9FefpIllfV/gvcDPSl8IZaTWXODucshy4KQyCmwW8LjtDcBK4EhJkyRNAo4EVpZ9T0qaVc51EnBVp35PRESMrpOjvyYDiyVNoCpey2x/X9K/S+qjun21Fji1tF8BHAMMAL8FTgawvVnSecDq0u5c25vL+seBS4F9qUZ9ZeRXREQXqRo4NX709/c7T9RH7NoO/6fDu53Cbu/GT9y4Q+0l3Wa7f7R2eaI+IiIak6ISERGNSVGJiIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYmIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMR0rKpJeJOlWSXdIWifp8yV+sKRbJN0n6XJJe5f4PmV7oOyf3nKus0v8XklHtcRnl9iApLM69VsiIqKeTl6pPA0cYftNwExgtqRZwBeAC23PAB4FTintTwEetf0a4MLSDkmHACcArwdmA1+TNEHSBOCrwNHAIcCJpW1ERHRJx4qKK0+Vzb3KYuAI4IoSXwzMLetzyjZl/3skqcSX2n7a9i+AAeDQsgzYvt/2M8DS0jYiIrqko30q5YpiLbARWAX8HHjM9rOlySAwpaxPAR4EKPsfB17ZGh92zPbiERHRJR0tKra32J4JTKW6snhdu2blU9vZt6PxbUiaL2mNpDWbNm0aPfGIiBiTnTL6y/ZjwHXALGCipD3LrqnAQ2V9EJgGUPa/AtjcGh92zPbi7b5/oe1+2/19fX1N/KSIiGijk6O/+iRNLOv7Au8F1gPXAseVZvOAq8r68rJN2f/vtl3iJ5TRYQcDM4BbgdXAjDKabG+qzvzlnfo9ERExuj1HbzJmk4HFZZTWHsAy29+XdA+wVNL5wE+BS0r7S4BvSBqgukI5AcD2OknLgHuAZ4HTbG8BkHQ6sBKYACyyva6DvyciIkbRsaJi+07gzW3i91P1rwyP/w44fjvnugC4oE18BbDiBScbERGNyBP1ERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGjMqEVF0ksk7VHW/0jS+yXt1fnUIiKi19S5UrkeeJGkKcA1wMnApZ1MKiIielOdoiLbvwWOBf7J9geo3gkfERGxlVpFRdLbgQ8DPyixTk6ZHxERPapOUTkTOBv4Tnm3yauoXrQVERGxlVGvOGz/GPixpJeU7fuBT3Y6sYiI6D11Rn+9vbytcX3ZfpOkr3U8s4iI6Dl1bn/9d+Ao4NcAtu8A3tnJpCIiojfVevjR9oPDQls6kEtERPS4OqO4HpT0DsCS9qbqT1nf2bQiIqIX1blSORU4DZgCDAIzy3ZERMRWRi0qth+x/WHbB9o+wPZf2v71aMdJmibpWknrJa2TdEaJ/42kX0laW5ZjWo45W9KApHslHdUSn11iA5LOaokfLOkWSfdJurxcSUVERJfUGf21WNLElu1JkhbVOPezwKdtvw6YBZwmaehJ/AttzyzLinLeQ4ATgNcDs4GvSZogaQLwVeBoqif5T2w5zxfKuWYAjwKn1MgrIiI6pM7trz+x/djQhu1HgTePdpDtDbZvL+tPUvXDTBnhkDnAUttP2/4FMAAcWpYB2/fbfgZYCsyRJOAI4Ipy/GJgbo3fExERHVKnqOwhadLQhqT92MFpWiRNpypEt5TQ6ZLulLSo5dxTgNZRZoMltr34K4HHbD87LB4REV1Sp6h8CfiJpPMknQf8BPiHul8g6aXAt4EzbT8BXAy8mqrDf0M5P4DaHO4xxNvlMF/SGklrNm3aVDf1iIjYQXU66pcAxwEPAxuBY21/o87Jy3tXvg180/aV5XwP295i+zngX6hub0F1pTGt5fCpwEMjxB8BJkrac1i83W9YaLvfdn9fX1+d1CMiYgzqvvnxZ8CVwFXAU5IOGu2A0udxCbDe9pdb4pNbmn0AuLusLwdOkLSPpIOBGcCtwGpgRhnptTdVZ/5y26aa2PK4cvy8kl9ERHTJqH0jkj4BLKC6UtlCddvJwJ+McujhwEeAuyStLbHPUY3emlnO8QDwMYAyA/Iy4B6qkWOn2d5ScjgdWAlMABbZXlfO91lgqaTzgZ9SFbGIiOiSOh3uZwB/XOfZlFa2b6B9v8eKEY65ALigTXxFu+PKjMmHDo9HRER31Ln99SDweKcTiYiI3lfnSuV+4DpJPwCeHgq29pNERERAvaLyy7LsXZaIiIi26rz58fMAkl5i+zedTykiInpV3vwYERGNyZsfIyKiMXnzY0RENCZvfoyIiMbkzY8REdGYEa9UyguyPmL7wzspn4iI6GEjXqmUubfm7KRcIiKix9XpU7lR0j8DlwO/f05l6K2OERERQ+oUlXeUz3NbYqZ6lW9ERMTvjdansgdwse1lOymfiIjoYaP1qTwHnL6TcomIiB5XZ0jxKkn/RdI0SfsNLR3PLCIiek6dPpW/Kp+tz6YYeFXz6URERC+rM0vxwTsjkYiI6H113lF/Uru47SXNpxMREb2szu2vt7Wsvwh4D3A7kKISERFbqXP76xOt25JeAXyjYxlFRETPqjX1/TC/BWaM1qiMFrtW0npJ6ySdUeL7SVol6b7yOanEJekiSQOS7pT0lpZzzSvt75M0ryX+Vkl3lWMukqQx/J6IiGhInTc/fk/S8rJ8H7gXuKrGuZ8FPm37dcAs4DRJhwBnAdfYngFcU7YBjqYqVjOA+cDF5fv3AxYAhwGHAguGClFpM7/luNk18oqIiA6p06fyjy3rzwL/YXtwtINsbwA2lPUnJa2nmj5/DvCu0mwxcB3w2RJfYtvAzZImSppc2q6yvRlA0ipgtqTrgJfbvqnElwBzgatr/KaIiOiAOkXll8AG278DkLSvpOm2H6j7JZKmA28GbgEOLAUH2xskHVCaTQFa3zA5WGIjxQfbxCMiokvq9Kl8C3iuZXtLidUi6aXAt4EzbT8xUtM2MY8h3i6H+ZLWSFqzadOm0VKOiIgxqlNU9rT9zNBGWd+7zskl7UVVUL5p+8oSfrjc1qJ8bizxQWBay+FTgYdGiU9tE9+G7YW2+2339/X11Uk9IiLGoE5R2STp/UMbkuYAj4x2UBmJdQmw3vaXW3YtB4ZGcM3j+U7/5cBJZRTYLODxcptsJXCkpEmlg/5IYGXZ96SkWeW7TqLeAIKIiOiQOn0qpwLfLC/qguoKoe1T9sMcDnwEuEvS2hL7HPD3wDJJp1D11xxf9q0AjgEGqIYtnwxge7Ok84DVpd25Q532wMeBS4F9qTro00kfEdFFdR5+/Dkwq/SNyPaTdU5s+wba93tA9VT+8PZm60krW/ctAha1ia8B3lAnn4iI6Lw6z6n8raSJtp8qQ4MnSTp/ZyQXERG9pU6fytG2HxvasP0o1W2qiIiIrdQpKhMk7TO0IWlfYJ8R2kdExDhVp6P+fwHXSPo61XMgf0X1JHxERMRW6nTU/4OkO4H3ltB5tld2Nq2IiOhFda5UAH4K7EV1pfLTzqUTERG9rM7orw8CtwLHAR8EbpF0XKcTi4iI3lPnSuW/Am+zvRFAUh/wb8AVnUwsIiJ6T53RX3sMFZTi1zWPi4iIcabOlcoPJa0ELivbH6KaUiUiImIrdUZ/fUbSscCfUk27stD2dzqeWURE9Jxao7/KtPVXjtowIiLGtfSNREREY1JUIiKiMdstKpKuKZ9f2HnpRERELxupT2WypD8D3i9pKcPejWL79o5mFhERPWekonIOcBbVu9+/PGyfgSM6lVRERPSm7RYV21cAV0j6b7bP24k5RUREj6rznMp5kt4PvLOErrP9/c6mFRERvajOhJJ/B5wB3FOWM0osIiJiK3UefnwfMNP2cwCSFlNNf392JxOLiIjeU/c5lYkt66/oRCIREdH76hSVvwN+KunScpVyG/C3ox0kaZGkjZLubon9jaRfSVpblmNa9p0taUDSvZKOaonPLrEBSWe1xA+WdIuk+yRdLmnvuj86IiI6Y9SiYvsyYBbV3F9XAm+3vbTGuS8FZreJX2h7ZllWAEg6BDgBeH055muSJkiaAHwVOBo4BDixtAX4QjnXDOBR4JQaOUVERAfVuv1le4Pt5bavsv1/ax5zPbC5Zh5zgKW2n7b9C2AAOLQsA7bvt/0MsBSYI0lUz8kMvShsMTC35ndFRESHdGPur9Ml3Vluj00qsSnAgy1tBktse/FXAo/ZfnZYvC1J8yWtkbRm06ZNTf2OiIgYZmcXlYuBVwMzgQ3Al0pcbdp6DPG2bC+03W+7v6+vb8cyjoiI2kYsKpL2aO1of6FsP2x7Sxme/C9Ut7egutKY1tJ0KvDQCPFHgImS9hwWj4iILhqxqJT/+N8h6aAmvkzS5JbNDwBDBWs5cIKkfSQdDMwAbgVWAzPKSK+9qTrzl9s2cC1wXDl+HnBVEzlGRMTY1Xn4cTKwTtKtwG+GgrbfP9JBki4D3gXsL2kQWAC8S9JMqltVDwAfK+daJ2kZ1RP7zwKn2d5SznM6sBKYACyyva58xWeBpZLOp3oY85I6PzgiIjqnTlH5/FhObPvENuHt/off9gXABW3iK4AVbeL38/zts4iI2AXUmVDyx5L+EJhh+98kvZjqqiEiImIrdSaU/M9Uz4P8zxKaAny3k0lFRERvqjOk+DTgcOAJANv3AQd0MqmIiOhNdYrK0+VpdgDKMN7tPhMSERHjV52i8mNJnwP2lfTnwLeA73U2rYiI6EV1ispZwCbgLqohwCuAv+5kUhER0ZvqjP56rkx5fwvVba97y8OHERERWxm1qEh6H/A/gJ9Tzbl1sKSP2b6608lFRERvqfPw45eAd9seAJD0auAHQIpKRERspU6fysahglLcD2zsUD4REdHDtnulIunYsrpO0gpgGVWfyvFUEz1GRERsZaTbX3/Rsv4w8GdlfRMwadvmEREx3m23qNg+eWcmEhERva/O6K+DgU8A01vbjzb1fUREjD91Rn99l2rK+u8Bz3U2nYiI6GV1isrvbF/U8UwiIqLn1SkqX5G0APgR8PRQ0PbtHcsqIiJ6Up2i8kbgI8ARPH/7y2U7Ypf0y3Pf2O0UxoWDzrmr2ynELqZOUfkA8KrW6e8jIiLaqfNE/R3AxE4nEhERva/OlcqBwM8krWbrPpUMKY6IiK3UKSoLxnJiSYuA/0Q1d9gbSmw/4HKqZ14eAD5o+1FJAr4CHAP8Fvjo0EAASfN4/v0t59teXOJvBS4F9qV6x8sZmZI/IqK7Rr39ZfvH7ZYa574UmD0sdhZwje0ZwDVlG+BoYEZZ5gMXw++L0ALgMOBQYIGkoSliLi5th44b/l0REbGTjVpUJD0p6Ymy/E7SFklPjHac7euBzcPCc4DFZX0xMLclvsSVm4GJkiYDRwGrbG+2/SiwCphd9r3c9k3l6mRJy7kiIqJL6rz58WWt25LmUl01jMWBtjeU826QdECJTwEebGk3WGIjxQfbxNuSNJ/qqoaDDjpojKlHRMRo6oz+2ort79L8Mypq91VjiLdle6Htftv9fX19Y0wxIiJGU2dCyWNbNvcA+hnhP+CjeFjS5HKVMpnnX/Y1CExraTcVeKjE3zUsfl2JT23TPiIiuqjOlcpftCxHAU9S9YGMxXJgXlmfB1zVEj9JlVnA4+U22UrgSEmTSgf9kcDKsu9JSbPKyLGTWs4VERFdUqdPZUzvVZF0GdVVxv6SBqlGcf09sEzSKcAvqd4iCdWQ4GOAAaohxSeX794s6Tyef9PkubaHOv8/zvNDiq8uS0REdNFIrxM+Z4TjbPu8kU5s+8Tt7HpPu5MBp23nPIuARW3ia4A3jJRDRETsXCNdqfymTewlwCnAK4ERi0pERIw/I71O+EtD65JeBpxBdVtqKfCl7R0XERHj14h9KuWJ9k8BH6Z6WPEt5SHEiIiIbYzUp/JF4FhgIfBG20/ttKwiIqInjTSk+NPAH1BN5vhQy1QtT9aZpiUiIsafkfpUdvhp+4iIGN9SOCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGdKWoSHpA0l2S1kpaU2L7SVol6b7yOanEJekiSQOS7pT0lpbzzCvt75M0rxu/JSIintfNK5V3255pu79snwVcY3sGcE3ZBjgamFGW+cDFUBUhYAFwGHAosGCoEEVERHfsSre/5gCLy/piYG5LfIkrNwMTJU0GjgJW2d5s+1FgFTB7ZycdERHP61ZRMfAjSbdJml9iB9reAFA+DyjxKcCDLccOltj24hER0SXbfUd9hx1u+yFJBwCrJP1shLZqE/MI8W1PUBWu+QAHHXTQjuYaERE1deVKxfZD5XMj8B2qPpGHy20tyufG0nwQmNZy+FTgoRHi7b5voe1+2/19fX1N/pSIiGix04uKpJdIetnQOnAkcDewHBgawTUPuKqsLwdOKqPAZgGPl9tjK4EjJU0qHfRHllhERHRJN25/HQh8R9LQ9/+r7R9KWg0sk3QK8Evg+NJ+BXAMMAD8FjgZwPZmSecBq0u7c21v3nk/IyIihtvpRcX2/cCb2sR/DbynTdzAads51yJgUdM5RkTE2OxKQ4ojIqLHpahERERjujWkuCe89TNLup3Cbu+2L57U7RQiokG5UomIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGpKhERERjer6oSJot6V5JA5LO6nY+ERHjWU8XFUkTgK8CRwOHACdKOqS7WUVEjF89XVSAQ4EB2/fbfgZYCszpck4REeNWrxeVKcCDLduDJRYREV2wZ7cTeIHUJuZtGknzgfll8ylJ93Y0q+7aH3ik20nUpX+c1+0UdiU99bcDYEG7fwXHrZ76++mTO/y3+8M6jXq9qAwC01q2pwIPDW9keyGwcGcl1U2S1tju73YesePyt+tt+ftVev3212pghqSDJe0NnAAs73JOERHjVk9fqdh+VtLpwEpgArDI9roupxURMW71dFEBsL0CWNHtPHYh4+I2324qf7velr8fIHubfu2IiIgx6fU+lYiI2IWkqOwmMl1N75K0SNJGSXd3O5fYMZKmSbpW0npJ6ySd0e2cui23v3YDZbqa/wP8OdUw69XAibbv6WpiUYukdwJPAUtsv6Hb+UR9kiYDk23fLullwG3A3PH8716uVHYPma6mh9m+Htjc7Txix9neYPv2sv4ksJ5xPqtHisruIdPVRHSZpOnAm4FbuptJd6Wo7B5qTVcTEZ0h6aXAt4EzbT/R7Xy6KUVl91BrupqIaJ6kvagKyjdtX9ntfLotRWX3kOlqIrpAkoBLgPW2v9ztfHYFKSq7AdvPAkPT1awHlmW6mt4h6TLgJuCPJQ1KOqXbOUVthwMfAY6QtLYsx3Q7qW7KkOKIiGhMrlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYkAJP2kRpszJb24w3nMHO05B0kflfTPDX9v4+eM8SlFJQKw/Y4azc4EdqiolNcS7IiZwLh+eC56W4pKBCDpqfL5LknXSbpC0s8kfVOVTwJ/AFwr6drS9khJN0m6XdK3yqSCSHpA0jmSbgCOl/RqST+UdJuk/y3ptaXd8ZLulnSHpOvLFDvnAh8qT2Z/qEbefZK+LWl1WQ6XtEfJYWJLuwFJB7Zr3/g/zBjX9ux2AhG7oDcDr6ealPNG4HDbF0n6FPBu249I2h/4a+C9tn8j6bPAp6iKAsDvbP8pgKRrgFNt3yfpMOBrwBHAOcBRtn8laaLtZySdA/TbPr1mrl8BLrR9g6SDgJW2XyfpKuADwNfLdz5g+2FJ/zq8PfC6F/jPK+L3UlQitnWr7UEASWuB6cANw9rMAg4BbqzmFGRvqvm7hlxejn8p8A7gW6UdwD7l80bgUknLgLHObvte4JCWc7+8vIHwcqqi9XWqCUYvH6V9RCNSVCK29XTL+hba/3siYJXtE7dzjt+Uzz2Ax2zPHN7A9qnlKuJ9wFpJ27SpYQ/g7bb/31bJSTcBr5HUB8wFzh+l/Ri+OmJb6VOJqO9JYOj/6m8GDpf0GgBJL5b0R8MPKC9s+oWk40s7SXpTWX+17VtsnwM8QvVOnNbvqONHVDNUU845s3yvge8AX6aalv3XI7WPaEqKSkR9C4GrJV1rexPwUeAySXdSFZnXbue4DwOnSLoDWAfMKfEvSrpL0t3A9cAdwLVUt6dqddQDnwT6Jd0p6R7g1JZ9lwN/yfO3vkZrH/GCZer7iIhoTK5UIiKiMemoj9hFSToZOGNY+Ebbp3Ujn4g6cvsrIiIak9tfERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGN+f+UQGoL+CmGNwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('interest_level');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每类样本分布不是很均匀"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "X_train = train.drop([\"interest_level\"], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "默认参数，此时学习率为0.1，比较大，观察弱分类数目的大致范围 （采用默认参数配置，看看模型是过拟合还是欠拟合）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {}\n",
    "param['gpu_id'] = 1\n",
    "param['updater'] = 'grow_gpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=3,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\software\\ProgramData\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYXGWZ9/HvXdVrOp3uLJ2QjYQdAkLAyCIojCwGxgFERCI4wjiivuKCOg6MDCCvK+O4IQOiAq+jiAgoiAgqq7gACZshEQiQDUIWkk7SSbrTy/3+8TzdqXSquqs7XX2qUr/PdZ2r6ix1zn2quutX5zmbuTsiIiIAqaQLEBGR4qFQEBGRHgoFERHpoVAQEZEeCgUREemhUBARkR4KBZEMZvYfZvbDpOsQSYpCocSY2UgzW2xm788YVm9mS83szIxhs8zsbjNbZ2bNZrbAzL5sZqPj+PPMrNPMWmL3spl9rMC1H2dmywu5jIHIVo+7f8Xd/7VAy1tsZicUYt6FMFyfV6m9L7s6hUKJcfcW4ALgO2bWFAdfBcx199sAzOytwEPAn4D93b0RmA10AIdkzO4v7j7S3UcCZwJXmdmhw7MmMhBmVpF0DVIm3F1dCXbATcDPgOOAN4CJGeMeBa7u5/XnAY/2GvY48P6M/lOB54BmQsgckDHugDisOU5zasa4U4AFwEbgVeBzQB2wBegCWmI3Kcd6XQP8Jr7+MWCvPN6P/YHfA2uB54GzBlMPcAXwk/i66YAD5wPLgHXAR4G3AM/Gdf9exnL2Ah6In8ca4KdAYxz3v3FZW+KyPp/He7wY+Pe4rDagIva/GtfleeD4LO/FkcDrQDpj2LuBZ+Pzw4G5wAZgJfDNHO/pccDyHOMagB8Dq4ElwKVAKo5LA/8d34NXgAvj+1iRY16LgRNyjPswsCh+rnd1/80ABnwLWAWsj+/RQbk+76T/X0upS7wAdYP84GA0sCL+452fMbwO6ASO6+f155ERCvGLrhnYN/bvC2wCTgQqgc/Hf86q2L8I+I/Y/474D7hffO0K4G0ZdR4Wn+f8ksmo46b4BXB4/BL8KXBLP6+pI3xpnx9fc1h8Xw4caD1kD4XrgBrgJKAV+BUwHpgcv5SOjdPvHd+vaqAJeAT4dsa8t/vy6+s9zpj+aWAqUAvsF9dzUkZ9WQMTeAk4MaP/F8DF8flfgA/E5yOBI3PMI+fnRQiEO4H6WMcLwIfiuI8SvpSnxPf7DwwiFOLf1Zr4eVYDVwOPxHHvBOYBjYSAOID4wyjX560uv07NRyXK3dcRfmGOAO7IGDWa0Cz4evcAM7sq7lfYZGaXZkx7ZBzeQthK+F/gxTjufcBv3P337t4OfIPwxfRWwi/RkcDX3H2ruz8A3A3Mia9tB2aY2Sh3X+fuTw5w9e5w98fdvYMQCjP7mf5dwGJ3v9HdO+Lybic0iQ1FPf/X3Vvd/XeEL/Gfufsqd38V+CNwKIC7L4rvV5u7rwa+CRzbx3z7eo+7fdfdl7n7FkLYV8d1qXT3xe7+Uo55/4z4eZhZPeHX888y3o+9zWycu7e4+18H8maYWTrWfom7b3T3xYQtgw/ESc4CvuPuy+Pf6dcGMv8M5wA3uPuT7t4GXAIcZWbT4zrUE7YQzd0XuvuKjPXbmc+7rCkUSpSZnUv4hfYH4OsZo9YRmikmdg9w98972K/wS8Iv6W5/dfdGD/sUdgMOBL4Sx00iNAt0z6OL8Ct1chy3LA7rtiSOA3gP4UtoiZk9bGZHDXD1Xs94vpkQQH2ZBhwRA67ZzJoJXyi7DVE9KzOeb8nSPxLAzMab2S1m9qqZbQB+AozrY759vcfdlmWMXwR8mrA1syoua1KOed8MnGFm1cAZwJPu3r2sDxG2Uv5uZk+Y2bv6qDGbcYQtxCUZwzI//0mZdfd6PhC9358WQtPc5PhD5HuEpsaVZna9mY2Kk+7s513WFAolyMzGE9pTPwx8BDjLzN4O4O6bCO3wZwxknu6+kvDr+p/ioNcIX7bdyzRCM8arcdxUM8v8+9k9jsPdn3D30whNLL8Cbu1ezEBqGoBlwMMx4Lq7ke7+sWGu56txnge7+yjgXELTRrfey+vrPc76Gne/2d2Pia9ztv9BkDndAsIX6snA+wkh0T3uRXefQ3g/vg7cZmZ1+a8mawi/xqdlDOv5/AnNN1Myxk0dwLwz9X5/6oCxbPs7+667v5nwY2Zf4N/i8Fyft+RBoVCavgf8yt0fjJvMnwd+EH8VEvv/xcwujgGCmU0B9sg1QzMbS9gZ+VwcdCvwj2Z2vJlVAp8l7Oz8MyF0NgGfN7NKMzuOECa3mFmVmZ1jZg2xSWQDodkDwi/ssWbWMETvQ7e7gX3N7AOxnkoze4uZHTDM9dQTdiI3m9lk4pdUhpXAnhn9fb3HOzCz/czsHfFzbiVspXRmmza6Gfgk8HbCPoXu+ZxrZk1xy6Q5Ds45HzOryewIW6K3Al+Oh0NPAz5D2DLqXq9PmdlkM2sk7BzvT2Wv5VTE+s83s5lxnb8CPObui+Pne0R83zbF96Ozn89b8pH0Tg11A+uA0wm/oBp7Db8f+HJG/xHAPYR/+mZgPvBlYGwcfx7hn6X7yJtVhDbn8RnzeDdhh+F64GHijts47sA4bH2c5t1xeBVwL6EZawPwBHBMxutuIDQBNJP76KMvZfQfRz87p+N0+xGOWFod5/8AYV/EgOoh+47miozpl5OxE5/wRXhpxnsyL76fTxO+5JdnTHsasDQu63N5vMeL2X7H9MGEfT8bCTvj7872HmZMvzvhC/w3vYb/JH7eLYQfAafneP1xcf17d3sT9l39JL7fy4DL2Hb0UQVhS/YNwtFHFxG2LCzHchZnWcaX4riPEnaad6/vlDj8eMIRRy1sO9JrZH+ft7r+O4tvsIhIQZjZycB17j6t34klcWo+EpEhZWa1ZnaKmVXEZrTLCQc5SAnQloKUBDN7G/DbbOM8HD0lRcLMRhCawvYn7Pf4DfApd9+QaGGSF4WCiIj0UPORiIj0KLmLbI0bN86nT5+edBkiIiVl3rx5a9y9qb/pSi4Upk+fzty5c5MuQ0SkpJjZkv6nUvORiIhkUCiIiEgPhYKIiPRQKIiISA+FgoiI9FAoiIhID4WCiIj0KJtQWLt+Ay+8/DJdnV39TywiUqbKJhTmffMM9v3xobS1bk66FBGRolU2oVC/X7h/+tY2hYKISC5lEwpWUQNAe1trwpWIiBSvsgmFVGW4ffHWti0JVyIiUrzKKBRqAWhXKIiI5FRGoRC2FNq3KhRERHIpn1CoCvsUOrSlICKSU9mEQroyhEJne1vClYiIFK+yCYWKuKXQqeYjEZGcyigUwo5mbSmIiORWRqEQthS6tKUgIpJT2YRCZXXYUujSloKISE5lFAojAOjq0BnNIiK5lFEohOYj36pQEBHJpWxCoSo2H3mnmo9ERHIpWCiY2Q1mtsrM5ucYb2b2XTNbZGbPmtlhhaoFoLomhALt2lIQEcmlkFsKNwGz+xh/MrBP7C4Ari1gLVRUVtPlBp1bC7kYEZGSVrBQcPdHgLV9THIa8GMP/go0mtnEQtWDGW1UgnY0i4jklOQ+hcnAsoz+5XHYDszsAjOba2ZzV69ePegFbqWSpavWDfr1IiK7uiRDwbIM82wTuvv17j7L3Wc1NTUNeoHtVsn0xopBv15EZFeXZCgsB6Zm9E8BXivkAtutEtM+BRGRnJIMhbuAf45HIR0JrHf3FYVcYLtVkerSIakiIrkUrC3FzH4GHAeMM7PlwOVAJYC7XwfcA5wCLAI2A+cXqpZuHVZJWucpiIjkVLBQcPc5/Yx34OOFWn42HVZFqkvNRyIiuZTNGc0AHalqKhQKIiI5lVUodKYqSXe1J12GiEjRKrNQqKbCtaUgIpJLWYVCV6pKoSAi0ofyCoV0FZUKBRGRnMosFKqpdO1TEBHJpaxCgXQ1lSgURERyKatQ6EpXU6XmIxGRnMoqFEhXUUUH4bw5ERHprbxCoaKGSuuko6Mj6UpERIpSWYVCy+J5ALRu2ZRwJSIixamsQqF+xokAtCkURESyKqtQSFWNAGDrlpaEKxERKU5lFQoV1TEUWrWlICKSTVmFQjqGgpqPRESyK7NQqAOgXc1HIiJZlVUoVNaEUOhs25xwJSIixamsQqGqdiQAHW3aUhARyaa8QqEmhIK2FEREsiurUKgeEUKha6tCQUQkm7IMBVcoiIhkVVahUFMbdjQrFEREsiurUKiorKbDU9CxJelSRESKUlmFAmZsoZolK1YnXYmISFEqr1AAtlo1ezaU3WqLiOSl7L4d26yaVKeaj0REsinDUKgh1dGadBkiIkWp7EKhPVVNhbYURESyKrtQ6EjXUNGlLQURkWzKLxRSNVQpFEREsipoKJjZbDN73swWmdnFWcZPM7P7zexZM3vIzKYUsh6AznQtlQoFEZGsChYKZpYGrgFOBmYAc8xsRq/JvgH82N0PBq4Evlqoerp1VtRQ7W2FXoyISEkq5JbC4cAid3/Z3bcCtwCn9ZpmBnB/fP5glvFDzitqqUahICKSTSFDYTKwLKN/eRyW6RngPfH5u4F6Mxvbe0ZmdoGZzTWzuatX79zZyF0VtdpSEBHJoZChYFmGea/+zwHHmtlTwLHAq0DHDi9yv97dZ7n7rKampp0qyitHUMtWOju7dmo+IiK7oooCzns5MDWjfwrwWuYE7v4acAaAmY0E3uPu6wtYE5uWPkPKnI2bW6ivH1XIRYmIlJxCbik8AexjZnuYWRVwNnBX5gRmNs7Mumu4BLihgPUAMOrAEwHY0lLQ7BERKUkFCwV37wAuBO4DFgK3uvtzZnalmZ0aJzsOeN7MXgAmAF8uVD3d0jVh62CzQkFEZAeFbD7C3e8B7uk17LKM57cBtxWyht4qRtQD0NbSPJyLFREpCWV3RnNlbQMAWzdvSLgSEZHiU3ahUF0XQ2GTmo9ERHor21DoaNWWgohIb2UXCiNGNgLQtUWhICLSW9mFQm19DIW2jQlXIiJSfMouFKrj0Ue0tSRbiIhIESq7ULBUmhavoW3ZU0mXIiJSdAp6nkKx2mIjGNE4MekyRESKTtltKQBsSY2gokPNRyIivZVlKLSlRlDRsSnpMkREik5ZhsLWijqqOxUKIiK9lWUotKfrqOranHQZIiJFpyxDobNyJLUKBRGRHZRlKHRV1jHCtyRdhohI0SnLUPDqeurYQpduySkisp2yDIWWZX+j0jrZuEmHpYqIZOo3FMxsLzOrjs+PM7NPmllj4UsrnIaDTwGgZf0bCVciIlJc8tlSuB3oNLO9gR8BewA3F7SqAqusGwPApubVCVciIlJc8gmFrni/5XcD33b3i4CSvkZEVf1YAFo3rE24EhGR4pJPKLSb2Rzgg8DdcVhl4UoqvJruUGhR85GISKZ8QuF84Cjgy+7+ipntAfyksGUV1sjGcQB0bNKWgohIpn6vkuruC4BPApjZaKDe3b9W6MIKqTsUOjetS7gSEZHiks/RRw+Z2SgzGwM8A9xoZt8sfGmFUzNyNF1uvLB4WdKliIgUlXyajxrcfQNwBnCju78ZOKGwZRVYKk0LtezPK0lXIiJSVPIJhQozmwicxbYdzSVvU6qeitqGpMsQESkq+YTClcB9wEvu/oSZ7Qm8WNiyCm9zehRV7RuSLkNEpKjks6P5F8AvMvpfBt5TyKKGQ1tFPdUKBRGR7eSzo3mKmf3SzFaZ2Uozu93MpgxHcYW0taqBuq6NSZchIlJU8mk+uhG4C5gETAZ+HYeVtM6qUYzs0gXxREQy5RMKTe5+o7t3xO4moKnAdRXcgnVp6mmhvaMz6VJERIpGPqGwxszONbN07M4F8ro+hJnNNrPnzWyRmV2cZfzuZvagmT1lZs+a2SkDXYHBmlHxKlXWSXOzTmATEemWTyj8C+Fw1NeBFcCZhEtf9MnM0sA1wMnADGCOmc3oNdmlwK3ufihwNvA/+Ze+cyrfdDoAzWtWDNciRUSKXr+h4O5L3f1Ud29y9/HufjrhRLb+HA4scveX3X0rcAtwWu/ZA6Pi8wbgtQHUvlOqG3YDYNM6hYKISLfB3nntM3lMMxnIvI7E8jgs0xXAuWa2HLgH+ES2GZnZBWY218zmrl49NPdAGDEmhELrupVDMj8RkV3BYEPBBjmN9+qfA9zk7lOAU4D/NbMdanL36919lrvPamoamn3cDWMnAdC+QaEgItJtsKHQ+8s9m+XA1Iz+KezYPPQh4FYAd/8LUAOMG2RNA1I/dgIAz7340nAsTkSkJOQMBTPbaGYbsnQbCecs9OcJYB8z28PMqgg7ku/qNc1S4Pi4vAMIoTAs98i0ylo2eQ0Hsmg4FiciUhJyXubC3et3Zsbu3mFmFxKum5QGbnD358zsSmCuu98FfBb4gZldRNj6OM/d89kKGRLr06OprKkbrsWJiBS9fq99tDPc/R7CDuTMYZdlPF8AHF3IGvqyqWI0tVt1noKISLfB7lPYJbRWjaGuQ6EgItKtrEOho3YsDV3NSZchIlI0yjoUnl1XyRg2sHFza9KliIgUhXwunZ3tKKRl8XLaew5HkYVy6Iz9SJuzZtWrSZciIlIU8tnR/E3C+QU3E05IOxvYDXgeuAE4rlDFFVrNmHAaRfPrS2H6XglXIyKSvHyaj2a7+/fdfaO7b3D364FT3P3nwOgC11dQo8ZPA2DTmqUJVyIiUhzyCYUuMzvLzFKxOytj3LCdU1AIjbuFUOhYp+YjERHILxTOAT4ArIrdBwgXsasFLixgbQVX07gb7Z6m9cWHki5FRKQo9LtPwd1fBv4px+hHh7acYZZKsy49hvraUf1PKyJSBvI5+mhKPNJolZmtNLPbzWzKcBQ3HDZUNlHXpiuliohAfs1HNxIuZDeJcD+EX8dhu4QttRNo7BiWa/CJiBS9fEKhyd1vdPeO2N0EDM1NDYrAcxtH0uRr2dLWkXQpIiKJyycU1pjZuWaWjt25wBuFLmy4vKnqdeqsjeUrhu1OoCIiRSufUPgX4CzgdWAFcCZwfiGLGk51x3wEgDeW/T3hSkREktdvKLj7Unc/1d2b3H28u58OnDEMtQ2LsbsfAMB9f/xLwpWIiCRvsBfE+8yQVpGg+t3C5S0mdKj5SERksKFgQ1pFkiprWZMax4yaXWY3iYjIoA02FEr68ha9NVdPpmHLsqTLEBFJXM4zms1sI9m//A2oLVhFCVi4dRxHdM2jtb2Tmsp00uWIiCQm55aCu9e7+6gsXb27F/TezsNt76p1jLdmXnl1RdKliIgkqqzvvNat4dhwXb+VLz2TcCUiIslSKABNex8KwJ//UtrX9xMR2VkKBaByzHRavYpZndpSEJHyplAASKVYVbsHjdaSdCUiIolSKETPtU9iWucS1m9uT7oUEZHEKBSi/avXMsGaeW7RS0mXIiKSGIVCNOHdXwLgjl/flXAlIiLJUShEI6a9mU43Dm/7U9KliIgkRqHQraqOl2x3JrAO913qKh4iInlTKGR4oWJfDraXeHHlxqRLERFJREFDwcxmm9nzZrbIzC7OMv5bZvZ07F4ws+ZC1tOftx57MqOthS/d9KskyxARSUzBQsHM0sA1wMnADGCOmc3InMbdL3L3me4+E7gauKNQ9eRjzIHvAOCAtqeTLENEJDGF3FI4HFjk7i+7+1bgFuC0PqafA/ysgPX0b/R0VjCOQzrn09ml/QoiUn4KGQqTgcybFCyPw3ZgZtOAPYAHcoy/wMzmmtnc1atXD3mhGQvi2YqDOcIWMG+xbrojIuWnkKGQ7e5suX5+nw3c5u6d2Ua6+/XuPsvdZzU1NQ1Zgdm8ffaZjLWNXHfzbQVdjohIMSpkKCwHpmb0TwFy3Qj5bJJuOopqZ5xMp6d4W9uDOjRVRMpOIUPhCWAfM9vDzKoIX/w7nC5sZvsBo4G/FLCW/I0Yw/z0ARxpC3hqWaIHQ4mIDLuChYK7dwAXAvcBC4Fb3f05M7vSzE7NmHQOcIsX0c/y/Ue1ckBqKV/+f3cmXYqIyLAq6G013f0e4J5ewy7r1X9FIWsYjOoP3UPnN/bnxLbfs37LOTTUViZdkojIsNAZzdnU78a8qlmclv4zp1/9cNLViIgMG4VCDoeffiETbS27Nz+uHc4iUjYUCrnsdzJbqObc9B846VuPJF2NiMiwUCjkUlFN9agJnJieR9ea52nv7Eq6IhGRglMo9CH1kQfpIM3H03dywje1b0FEdn0Khb7UjSM9aiKnp/7EiLULOf2aR5OuSESkoBQK/bCPPUqHpbmy8kbmv9qsnc4isktTKPSndjRVp36Lt6Re4N32MMd8/cGkKxIRKRiFQj5mnotXj+Lyih9j65dy/H8/lHRFIiIFoVDIRyqFXfAQdal2bqr8OstXr+Okb2nHs4jsehQK+Rq7F3b2zeydeo2rK6/mhZUbOVFHJInILkahMBD7zYaG3TkpPY9vVFzHi6s2st+lv026KhGRIaNQGKhPPwv1Ezmz4o98tvJ22jq62OcL9+jkNhHZJSgUBsoMLloAIyfwifQdfKLyTto7nX2+8FtO/57OYxCR0qZQGIxUCj6zEOqa+Gz65/yy6j9J0cXTy9ez/3+qOUlESpdCYbBSafjs81A/iUNTL/FY7ScZQSut7V3scfFvOON//pR0hSIiA6ZQ2BmpNHx2IZzyDZp8LfPrP8G+6ddx4Mmlzez/n7/VGdAiUlIUCkPh8A/DObeRat/M7yo/w0Vj/wp42Gq45B5mf/sRhYOIlAQrtS+rWbNm+dy5c5MuI7sNr8E1R0LbenzEOA7f8DVWd4zoGV1TkeKZK06iuiKdYJEiUo7MbJ67z+pvOm0pDKVRk+DfX4HjL8c2r+GJig/zwofrqakMb3NrRxf7XXove1z8G069WkcqiUjx0ZZCobw6D354AngX1I7Gz7mdk2/bxN9f37jdZNUVKR7/wgk01FYmVKiIlIN8txQUCoXU1gKPXQcPfAlwqBkN597GqlEH8barHqStY/sT3lLAm6eN5hcfe2si5YrIrkuhUEzaNsK1R0PzktBf0wjvvxWfejjvuvpRFry2gd6fQk1linmXnkhddcWwlysiux6FQjFqa4nhsDj01zTCnFtg2lF0djkHXX4vW9p3vFyGATMmjeKuC48hnbJhLVlEdg0KhWLW1gJzfwS/vxxwsDScfi0ceDpUVIeAuOI+tmztzPpyAw6e0sCdFx4zrGWLSOlSKJSCrZvgyR/Dff8RdkhjcMxFMOt8aNy9Z7JVG1t5+1UP0pplK6JbCnjTlAZ+9fGjMdPWhIhsT6FQSrq64OUH4fZ/hS1rwzBLwylXwYzToW5cz6TuzkGX38emHFsRvY2oSvPUZSfq3AiRMqdQKFXNS+GZn8NDX4lbD4SjlmZ/Ffb/R6gZtcNLtmzt5M1f+j2b8wwK2LZlcdvH3kplWqeriOzqFAqlzh1Wzoebz4YNy7cNHzEW3vVt2OckqKzJ+fKzrvszcxevY7B3eUgBh+7eyO3/5+hBzkFEiolCYVfiDsseh198EDau2Da8dgwcfxnsfQI0Ts1rVp1dzsFfvI9NbflvVfRlZHWa+V+cPSTzEpHCKYpQMLPZwHeANPBDd/9almnOAq4AHHjG3d/f1zzLMhQydXbA4kdg4d0w9wboOcPBYOY5MP2Y0OUZEr2dee2feXLJ4Lcw+qIAEUlO4qFgZmngBeBEYDnwBDDH3RdkTLMPcCvwDndfZ2bj3X1VX/Mt+1DI5A6r/w4vPwwPXwVb3tg2rqIaDnrvTodEpoMuv5eWIdrCyFcKeMseY/j5R44a1uWK7GqKIRSOAq5w93fG/ksA3P2rGdNcBbzg7j/Md74KhT50dcGqBbD40R1DAoOD3gOTD4NJh8HEQ6BqRM5Z7YwkwmMw6msq+NsV70y6DJFhUQyhcCYw293/NfZ/ADjC3S/MmOZXhK2JowlNTFe4+71Z5nUBcAHA7rvv/uYlS5YUpOZdTlcXrF4YQmLxo7Dw15B5QQ1LhSanyYfBbofAhBlQWTusJb7pivvY2NoxrMtMkoJIklIMofBe4J29QuFwd/9ExjR3A+3AWcAU4I/AQe7enGu+2lLYSRtXwmtPhqu4Pv4DaO31VlsKDn5f2JKYeAhMOCjrYbDFoNwCJQnpeB5kpyvQSl0xhEI+zUfXAX9195ti//3Axe7+RK75KhSGmHu4UN+KZ2HFM/DEj6B13fbTWCqcRDfx4BAUux0CdWOTqXeIve/7f2Hu4rV0ltZBeFLkijFAiyEUKghNQ8cDrxJ2NL/f3Z/LmGY2YefzB81sHPAUMNPd38g2T1AoDJuNr4eQWPEsrHga/v4b6H0t15rG0Pw0/gAYPwOa9oXq+kTKLQUKINlZ+00YyX0XHTuo1+YbCgW7LrO7d5jZhcB9hP0FN7j7c2Z2JTDX3e+K404yswVAJ/BvfQWCDKP63UK3b8avnc1r4fVnQ1CsWggL74K/XrPja2saQxPU2L1h7F6ha5gKqfK+1MaudgSVmu+GX2dX4X9R6OQ12TldnbBucQiJNc/D6hfg73fD1pYdp62ohb3+AcbsGYJizF4hOOonQkqX2hAppMS3FKRMpNLbtgZ417bh7tCyEt54Cda+FB7fWAQvPQDP37PjfCwFe70DGqfB6OkZ3TSoaRiedRERhYIUiNm2Jqjpva6f1NUFG17dFhbrXoF1S8IO70V/yDG/FMw4LTRDNe4eH6eGxyI9OkqkFCkUZPilUuELvXEq7HncjuNb14eQWLd4+27BnduuHNubpWDf2SEkGqZAw2QYFR9H7gZp/amL5EP/KVJ8ahri4a8H7ziuqws2rYb1y8Jlxtcvg+Zl4fHlh6B9c+75Vo2EPd4et2AmhcdRE8M+jfqJUDs6bOGIlDGFgpSWVArqJ4RuSo59Zq3rYf2roYlq/TLYsAI2vhYOs33h3txbGxCuGTXpsO2Do37i9uFRoMuDiBQDhYLsemoaQjdhRu5p2luh5fUQFBtiYHQHx/w7wPu4dpOlYdw+2wKju8sMjpHjIV059OsmUmAKBSlPlTXbjnDq7T3x+ozu0LYhe3D87fZwhdq+1I3fPijqJ8Ymq4wtkNoxOhxXiopCQSQXs21bHU37bT/ulP/a9ryrCzav2TE4Nq4ITVeOeP2IAAAJ6ElEQVQv/q7vJissnBFeNw7qmmKX8Xzk+G2Pw3zBQik/CgWRnZVKhS/skeP7nq6jLYZFDIyNr4ed5ptWw6Y1IVieu6OfAAEqasI1qHoCYzyMbIIR48LtWuvGxedjyv4schk4hYLIcKmoDifjjZ7W/7TtrSEkWlaF0GhZBZtWQcvq+LgqnATYX4BAOJN80szQVDWiuxsb+8du31/bqCApcwoFkWJUWRPPt5jS/7Sd7bD5jW1bG5vWhOtU9TyP/S8/2Pchu5ksBZNnZYTGmOwhMmJMuNZVRdXOra8UDYWCSKlLV247ezwf7uHaVJvXhjDZsjY+z+x/I/S/8gi0b8q/lqq6ECa1o7d13cHRMyzjufaRFB2Fgki5MQuXOK+uz68pq9vWzTuGxpZ12buFd+XXtNVTUwqmHhGCoidAMsKj97CaBjVzFYhCQUTyUzUidI1T83+NO7RtDEHR2pwRHBnPe4Y3939yYTaWgmlHh6CobQwBUtO47XltYzyKLON5RfXAllFGFAoiUjhm4YKFNaOAAWyVAHRsjYGRLUC6+zeEYUv+HM4pGXB9KZhyeB8B0tgrbBpCE9kufDkUhYKIFKeKqvwO9c2mY2u43El3qLQ2h/6eYGnefny/55JkYalwaHB3WPR0o6A6PtY0QHUMxeru/vqibvpSKIjIrqeiKpy7MbJp4K/t6oyB0StUtjSHrZHMQOmebulfoWPLABdk4Va23aGRLUAyh9U0hLPhC3x/EYWCiEimVHrbYbiD0dEWmrXaYtNWz/Pux/W9nq+HJX+CrXkc5TVmL/jkk4OrK08KBRGRoVRRPfitFAg759u39AqSGC4TDxnaWrNQKIiIFBOzbUd65XvuyRDS5RlFRKSHQkFERHooFEREpIdCQUREeigURESkh0JBRER6KBRERKSHQkFERHqYuyddw4CY2WpgySBfPg5YM4TlJEXrUVy0HsVF65HdNHfv9zTrkguFnWFmc919VtJ17CytR3HRehQXrcfOUfORiIj0UCiIiEiPcguF65MuYIhoPYqL1qO4aD12QlntUxARkb6V25aCiIj0QaEgIiI9yiYUzGy2mT1vZovM7OKk6+mLmd1gZqvMbH7GsDFm9nszezE+jo7Dzcy+G9frWTM7LLnKtzGzqWb2oJktNLPnzOxTcXiprUeNmT1uZs/E9fhiHL6HmT0W1+PnZlYVh1fH/kVx/PQk6+/NzNJm9pSZ3R37S249zGyxmf3NzJ42s7lxWEn9XQGYWaOZ3WZmf4//J0cVw3qURSiYWRq4BjgZmAHMMbMZyVbVp5uA2b2GXQzc7+77APfHfgjrtE/sLgCuHaYa+9MBfNbdDwCOBD4e3/NSW4824B3ufggwE5htZkcCXwe+FddjHfChOP2HgHXuvjfwrThdMfkUsDCjv1TX4x/cfWbGcfyl9ncF8B3gXnffHziE8Lkkvx7uvst3wFHAfRn9lwCXJF1XPzVPB+Zn9D8PTIzPJwLPx+ffB+Zkm66YOuBO4MRSXg9gBPAkcAThTNOK3n9fwH3AUfF5RZzOkq491jOF8EXzDuBuwEp0PRYD43oNK6m/K2AU8Erv97QY1qMsthSAycCyjP7lcVgpmeDuKwDi4/g4vOjXLTY9HAo8RgmuR2xyeRpYBfweeAlodveOOElmrT3rEcevB8YOb8U5fRv4PNAV+8dSmuvhwO/MbJ6ZXRCHldrf1Z7AauDG2Jz3QzOrowjWo1xCwbIM21WOxS3qdTOzkcDtwKfdfUNfk2YZVhTr4e6d7j6T8Ev7cOCAbJPFx6JcDzN7F7DK3edlDs4yaVGvR3S0ux9GaFL5uJm9vY9pi3U9KoDDgGvd/VBgE9uairIZtvUol1BYDkzN6J8CvJZQLYO10swmAsTHVXF40a6bmVUSAuGn7n5HHFxy69HN3ZuBhwj7SBrNrCKOyqy1Zz3i+AZg7fBWmtXRwKlmthi4hdCE9G1Kbz1w99fi4yrgl4SgLrW/q+XAcnd/LPbfRgiJxNejXELhCWCfeKRFFXA2cFfCNQ3UXcAH4/MPEtrou4f/czw64UhgfffmZ5LMzIAfAQvd/ZsZo0ptPZrMrDE+rwVOIOwQfBA4M07Wez261+9M4AGPjcBJcvdL3H2Ku08n/P0/4O7nUGLrYWZ1Zlbf/Rw4CZhPif1dufvrwDIz2y8OOh5YQDGsR9I7XIZxx84pwAuE9uAvJF1PP7X+DFgBtBN+IXyI0J57P/BifBwTpzXCkVUvAX8DZiVdf6zrGMLm7bPA07E7pQTX42Dgqbge84HL4vA9gceBRcAvgOo4vCb2L4rj90x6HbKs03HA3aW4HrHeZ2L3XPf/cqn9XcXaZgJz49/Wr4DRxbAeusyFiIj0KJfmIxERyYNCQUREeigURESkh0JBRER6KBRERKSHQkFERHooFETyYGYzzeyUjP5TbYguwW5mnzazEUMxL5GdpfMURPJgZucRThi6sADzXhznvWYAr0m7e+dQ1yKiLQXZpZjZ9HjDkh9YuCnO7+LlKbJNu5eZ3RuvtvlHM9s/Dn+vmc23cGOdR+KlUa4E3hdv7PI+MzvPzL4Xp7/JzK61cFOhl83sWAs3SlpoZjdlLO9aM5tr29+s55PAJOBBM3swDptj4SYy883s6xmvbzGzK83sMeAoM/uamS2IN135RmHeUSk7SZ/qrU7dUHaE+1B0ADNj/63AuTmmvR/YJz4/gnB9HwiXEZgcnzfGx/OA72W8tqefcFOkWwiXIjgN2AC8ifCja15GLd2XLEgTLqx3cOxfTLw/ACEglgJNhCtpPgCcHsc5cFb3vAjX1LfMOtWp29lOWwqyK3rF3Z+Oz+cRgmI78ZLebwV+Ee+V8H3CTU0A/gTcZGYfJnyB5+PX7u6EQFnp7n9z9y7C9Xm6l3+WmT1JuJbSgYS7APb2FuAhd1/t4T4GPwW6Lw3dSbjqLITgaQV+aGZnAJvzrFOkTxX9TyJSctoynncC2ZqPUoQbzMzsPcLdP2pmRwD/CDxtZjtM08cyu3otvwuoMLM9gM8Bb3H3dbFZqSbLfLJdN79bq8f9CO7eYWaHE66ueTZwIeFy2CI7RVsKUpY83PDnFTN7L/TcGP2Q+Hwvd3/M3S8j3IZyKrARqN+JRY4i3EhlvZlNINwgplvmvB8DjjWzcRbuLT4HeLj3zOKWToO73wN8mnDFTZGdpi0FKWfnANea2aVAJWG/wDPAf5nZPoRf7ffHYUuBi2NT01cHuiB3f8bMniI0J71MaKLqdj3wWzNb4e7/YGaXEO5zYMA97n7njnOkHrjTzGridBcNtCaRbHRIqoiI9FDzkYiI9FDzkezyzOwawj2KM33H3W9Moh6RYqbmIxER6aHmIxER6aFQEBGRHgoFERHpoVAQEZEe/x+QGeKqPQ1ZggAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\software\\ProgramData\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1oAAANGCAYAAADktv9+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xu4pXdZH/zvnUwHzcgMp2IcOeSlKJpa5dgCVUB4aZCIVmuLUFqiohValGLlJSmNY4QBe+AkKOVVQdR6qAcsRAyI0mIIIkfBhIOWACE0gMgeICQD4dc/nrWSlZ2196y99rPOn891rSuzn/WsZ//W3nsm8537/t1PtdYCAABAf05b9AIAAADWjaAFAADQM0ELAACgZ4IWAABAzwQtAACAnglaAAAAPRO0AAAAeiZoAQAA9EzQAgAA6JmgBWyMqvrFqrq+qv7emOeeXlWtqh617fjhwXN/WlWfrqovVNU1VfUHVfXYqrrVyLlnDa4x+jhRVe+qqqdU1enzeJ+7qaonVdV5i17HflTVGVV1rKoeMua58wZf97MWsK6jg3Xdc96fe5Gq6uVV9dlFryNJBl//VlV3WPRaAAQtYJM8Jcn/SfJLVfW3hgcHwesnk7y8tfaqkeNfk+QdSf59kj9J8i+TPDTJk5N8NMkvJnnGmM/zM0keMHj8sySXJnlekv/Y/1vasyclOW/Ri9inM5L8RJKHjHnu4nRf94/Nc0EDR9Ota6OCFgDjHVj0AgDmpbV2oqp+IMlr0wWknxgErl9Ock26IJYkqaoDSV6Z5HZJ/n5r7Yptl/vNqrooyb3GfKoPt9bePPLxH1TVNyR5TJIf6+0NcQuttU8k+cSi19GnqjqjtXbtotcBwN6oaAEbpbX2h0lekuSCqrpPkmNJvinJD7TWtkZO/a4kZyd51piQNbzWh1prr5zwU28l+cLogao6raqeVlXvHbQ0fryqXlFVd9r+4qr6/kEL4nVV9amq+t2q+vpt59ytqn69qq4eXO+aqnr9sJWtqq5M8neTPHiktfHK3RY9OOdFVfUvquqKqrp2sI5vn/B9j17rYFU9Y+T9fqKqXlZVf3vbeQ+tqjdU1V9X1eer6sNV9duDlsGzclOQ+omR9/HywWtv0To4uNZ7quoBVfWmwTWvrKrvGzx/blW9ffDe3l1Vj9i2nrsP1vmBwTkfrapXjbagDtoY/2zw4ctG1nVs5JzvqKrLBtf4TFW9rqoesO1zDVvf7l1Vv1VVf5PkrwbP7fr93eFr/pTB9e4+5rmfrqqTwza7qrpXVb168HN4/eDzXDzu53EaVfXNg/V+ZvA1eFNVnbvDeZcNftY/WlU/VVVP2P593edavqGqfq+q/mbwed5ZVY/fds5pg5/X9w1+Zj5dVX9eVT86cs7frqqXVtVHRn6mL62q/7ePdQKrTUUL2EQ/nuScJL+V5M5JXtJae922cx4++O//mOL6p1VXEUuSI0m+M8kjkvz0tvN+LskPJXlRklcnOSvJTyV5SFXdu7X2ySSpqvOTHE/ya0nOT3L7dAHxsqq6X2vtA4Pr/X6S05M8LcmHk9whyQOT3Gbw/HcN3vNWuhbCJLl+gvdzbpL7JbkwyWcH1//dqrpHa+1/T/D6VNVpSX4vybeka6F8U5K7pmvZfENV3be19vnBX6QvTvLGJN+f5NNJvjrd1+9gupbARyT5gyS/kOTnB5/iVFWsM5O8bPC5r0rX/vmLVXXnJN+T7uu7NXiPr6yqu7XWrh689miSv07y9MHnuV2Sxyf506q6V2vtfUnenuT7Bp/jmYP3kMHnSlU9NsmvpqumPibJrQZfxzdU1cNaa3+ybb2/k+TX0/2jwKHBsVN9f8f5lXQ/d+dlpM21uv2Cj0vyqtbaJ6vqUJLXJflgkn+drsJ7ZpJvTXLrXa4/kap68OD6f57kB9L93D0pyauq6jGttd8YnPeNg/Pen+5rfG2SHx6stRdVdY90P38fT/Ij6b63j0vy8qr6ytbasMX3ael+nz0zyf9K8reSfF1u/vX+5ST3Ttde/P7Bc/dO93sU2HStNQ8PD4+Ne6T7y25L9xf3rxjz/GsGz99q2/FK949Uw8fpI8+dNXjNuMfLtp37dYPjL952/b8/OP6swce3SfeXzYu3nXfnJNcl+dXBx7cfvO5HT/G+35PkDXv4OrV0+9puPXLsK5PckOTpe7jO9w6u9d3bjt93cPyJg4//yeDjb9rlWncYnHNszHPnDZ47a+TYGwbH7jNy7HZJvjj42h4dOf5Ng3OfvMvnPz3dX7rfn+S5Y97LedvOPy3dnr4/T3LayPGvSBdoLh05dmxwjZ/cdo2Jvr87rPe3k3xk2+f+tsH1vn3w8X0GH3/nFNd/eZLPnuKcywbv9StGjp2e5N2DtdXg2G+mC/N32Pb1+4vt39cdPs/w63eHXc75tcHvnTtvO/77ST6X5Mjg41cleccpPt9nkjxvr18zDw+PzXhoHQQ2zqC68uQkX0pyx3R/uZ7Uj6ZrARw+3jXmnBekqwDdL11F4IJ0QzF+beScbx389+WjL2ytvSXJFUkeNjj0gCRfPua8jyT5o5HzPpWuxezHq+qpgzawvv6M/+PW2mdGPvc16aoBd93DNb49XXXqVVV1YPhI8s50Qe4hg/PemeRkkpdW1eOr6m59vIEkH2utvW34QWvtU+newzvbTZWrpPvaJyPvbbDWC6rq8qo6mS6gnUzyNUlu1r65g3ukq4r9cmvtSyNr+Gy6EHT/qjpj22t+e9vH+/n+vizJnZKMtrN9X7qv+2sGH/9lkr9J8tNV9cNVdfaE1z6lQbXsHyT5rcF7TpK01m5IVxG6U7qvUZI8OMkftUE1d3Del9IFsL48NMnrB7+HRr083aCVYTvnW5J8U1X9bFWdU1WHx1zrLUnOG7QY3r9GhuwACFrAJvp36f4y9dgkH0jXQvbl28758OC/28PEf8tNIertO1z/qtbaWwePN7TWnp2uJfCfVtU5g3OGrUXjpuNdPfL8ROe11lq60HVJupantyf5RFW9sKr22/r112OOXZ8uAE7qK9NV507m5kH1C+la1O6QJK21v0oXCD6e5MVJ/qqq/mp0X8yUPjXm2Mntx1trJwe//LKRw89N9/17ZZJHpQsN90sXsif5Gpzqe3hakttuO36zc/f5/X3N4HrDPWm3TfIdSV4xCDtp3f7EB6cLuseT/MVgj9ZP9hAebpuuErzT+09u/vN+zZjzxh2b1u0nXMuz0/1Zcf90X8O/Huwxu+/Iax6d5JeSPCFd1e5T1e2zPLPH9QIrStACNsrgX+ovSveXzN9I12p29yTP2nbqcM/Wd4webK19fBii0rUNTerPB/8dVs+G4eWrxpx7NMkn93heWjec4wdaa2emqxA8L90+mP+0h3XOyifTvZf77fAY7hlLa+2NrbVHpdvfdv90f4F9flV977wXPfC4dD8vF7TWLmmtvWXw/Z/0Xk2n+h5+KV01aVTbfuK039+RytE/rqrbpPsHhlulq3SNnvfu1tr3pgsa90zyG+n2rO13UubfpHuPO73/5OY/71855rw+g8tfT7KW1toXW2vPba3dO12r6WPStexeMqxAttY+2Vp7SmvtrHT/KHN+ku/Otgo0sJkELWBjDFrVfindX6R+NElaN4b9uUl+tKr+4cjpv5vk8nTTCb+uh08/nAz38cF//2jw35tt8q+q+6VrR3v94NBlST4/5rw7ZdACNe6Ttdbe31p7Zro9MPceeWqvlai+vDrdX+BPH6n2jT7et/0FrbUbWmt/mm44Q3LT+xgO8JjX+2jZNjRkMC3vq7edt9O63pduj9Zjq6pGrnEo3Z60y9oex7fv8v3dycvSVekek+4fFy5rrb13h2u31tq7Wmv/Nl275yTX322tn0vyp0m+e7RyPGh9fFy6gSHvHxz+n0keWiM3HB6c90/3s4ZtXj/4HEe3Hf+X6fbsvXn7C1prn26t/Va6Kuvt0u3H3H7Oh1trL0r3jzT7+poB68HUQWCTnJ9uYMG3tdY+PXL8P6RrCfvFqrpna+3zrbUbquofp2vVektV/f/phir8TboWuH+Qrjo1bvT7Xarq/oNfH0rXpnh+kg+lmyaX1tr7quqlSZ5cVV9K15p0VroWtY+kq1aktfbpqvqpJMer6hXp9nndPt2Nca9LN7VvOK3tRUn+e7p2yJPpgtg3JnnOyNreneR7q+rRSf53kutaa+/eyxdxSr+e5J8n+f2qekG6vS1fSLc/51uT/F5r7Xer6ocH6744Xfvml6WbPpgkf5gkrbXPVNWHknxnVb0+XfvfJ1trV85o7a9Otw/nvekqk/dJN7nyqm3n/VW6UPzPq+qKdEMdrm6tXV1VT0s3dfDVVfVf01WUfjzdz9LTT7WAPXx/x2qtvbeqLkv3c3jndNMuR6//7emqY69M93NR6Sozt8lN1d3dnF5V3zPm+Odaa68ZfN7XJfnjqvrPg/U/Kck3JHnMoDUy6SrLj0ry+qp6Vrqv5w/npsmLX8pkHlVVt6g4D8LST6bbM/jH1d0L71PpfjbPTfK0QRtlqupV6YbHvDXdtMm7prvX3oeSfKCqjiT543TtxO9NV+G+X7qpmL8z4TqBdbboaRweHh4e83ikC0Unk7x0h+fvn26S3nO3HT+c7i+Jb8lN98K6Jt2Y7iclOWPk3LNyy2mDn09X0XhekjO3Xfu0dPtt3jdY2ycyGA4wZn0/kG5P0PXpqgyvTHL2yPN3TFe1GP4F/zOD85+Sm087vGu68HhisL4rT/F1a0leNOb4lUlevsfvwYF0bWjvHHxdPjNY70uS3H3k+/A7g+tfl676+IYkj9p2rYel26d03WCNLx8cPy/jpw6+Z4f38OpTved0YePnB9/3z6UbPf/Ng+u+Ydtrv3fwnk5m22TEdGP+3zx4759NFxwfuO31xzJmat6k399TfP1/cHDta5Mc3vbcPdIFhr8cPP/pdFWox09w3Zdn52mbV46c983pqkmfHXyOyzKYerjtet88+Dpdl24v1X9M9/ukZTARcJe1HNtlLW3kvG9Id+uGT6f7PfXO3HJa5FOTXJru9+X16QLWzye56+D5W6W7RcO70v3ZcG26wHUsI38ueHh4bO5jOE4VAGApVdVr04Xnr130WgAmpXUQAFgaVfXcJO9I10J7u3RtfQ9PV9UFWBmCFgD7MhgyspsvtZH7R8EpnJ5uMuiZ6Vr+Lk/yL1prv7LQVQHskdZBAKZWVWcl+eApTvvJ1tqxmS8GAJaIihYA+3F1uklrpzoHADaKihYAAEDP3LAYAACgZ1oHx6iqSnI03X1KAACAzXbrdDehn7gdUNAa72iSqxa9CAAAYGncKclHJz1Z0BrvM0nykY98JIcPH170WgAAgAU5ceJE7nznOyd77HYTtHZx+PBhQQsAANgzwzAAAAB6JmgBAAD0TNACAADomaAFAADQM0ELAACgZ4IWAABAzwQtAACAnglaAAAAPRO0AAAAeiZoAQAA9EzQAgAA6JmgBQAA0DNBCwAAoGeCFgAAQM8ELQAAgJ4JWgAAAD0TtAAAAHomaAEAAPRM0AIAAOiZoAUAANAzQQsAAKBnghYAAEDPBC0AAICeCVoAAAA9E7QAAAB6JmgBAAD0TNACAADomaAFAADQM0ELAACgZ4IWAABAzwQtAACAnglaS+zak1/MWU+/OGc9/eJce/KLi14OAAAwIUELAACgZ4LWijj7wktUtgAAYEUIWgAAAD0TtAAAAHomaK2Ysy+8RPsgAAAsOUELAACgZ4IWAABAzwStFWQCIQAALDdBCwAAoGeCFgAAQM8ErRVmAiEAACwnQQsAAKBngtaKU9UCAIDlI2gBAAD0TNBaA8a9AwDAchG0AAAAeiZorRH7tQAAYDkIWgAAAD0TtNaMqhYAACzeUgStqnpSVX2wqq6rqrdV1bec4vzbVNWLq+pjg9dcUVWPHHn+/Kr6s6r6TFV9vKpeWVX3mP07AQAAWIKgVVWPTvL8JM9Kcq8kb0zymqq6yw7nH0zyuiRnJfmeJPdI8oNJPjpy2oOTvDjJ/ZM8PMmBJK+tqkOzeRfLxRRCAABYrAOLXkCSpyb5hdbazw8+fkpVnZPkiUnOH3P+9ye5XZIHtta+MDj2odETWmuPGP24qr4vyceT3CfJ/+px7QAAALew0IrWoDp1nySv3fbUa5M8cIeXfUeSy5K8uKquqar3VNUFVXX6Lp/qyOC/n9phHbeqqsPDR5JbT/4ulpf9WgAAsBiLbh28Q5LTk1yz7fg1Sc7c4TV3S9cyeHqSRyZ5ZpIfS/Lvx51cVZXkuUn+pLX2nh2ueX6SrZHHVZO/BQAAgJtbdNAaats+rjHHhk5L1wb4Q621t7XWfj3d/q4n7nD+i5J8Y5LH7PL5n52u6jV83GnCdS89+7UAAGD+Fr1H65NJbsgtq1d3zC2rXEMfS/KF1toNI8euSHJmVR1srZ0cHqyqn0nXavig1tqOVarW2vVJrh953Z7eBAAAwKiFVrQGoeht6SYDjnp4kjft8LJLk9y9qkbX/rVJPjYMWdV5UZLvTvLQ1toH+1356rFfCwAA5mcZWgefm+QJVfX9VfX1VfW8JHdJ8pIkqapXVNWzR87/uSS3T/KCqvraqjo3yQXpxrkPvTjJ45I8NslnqurMwePL5/GGAACAzbbwoNVa+40kT0lyYZJ3JnlQkke21oYj2++S5KtGzv9Ikn+U5H5J/jzJC5O8IMlzRi77xHR7rd6QrtVw+Hj0DN/K0lPVAgCA+Vj0Hq0kSWvtZ5P87A7PPWTMscvS3Yx4p+vZZAUAACzMwitazJcphAAAMHuCFgAAQM8ErQ1lvxYAAMyOoLXBhC0AAJgNQQsAAKBngtaGMxwDAAD6J2gBAAD0TNACAADomaBFEoMxAACgT4IWAABAzwQtbmQwBgAA9EPQ4ha0EQIAwP4IWkvsjIMHcuVzzs3lF52z6KUAAAB7IGgxlqoWAABMT9ACAADomaDFjgzHAACA6QhanJI2QgAA2BtBawWccfCAgRgAALBCBC0moo0QAAAmJ2gBAAD0TNBaEctyTy37tQAA4NQELQAAgJ4JWitmGQZjqGoBAMDuBC2mYjgGAADsTNBaQctQ1RpS3QIAgFsStAAAAHomaK2oZZlCmKhqAQDAdoIWvbBnCwAAbiJorbhl2q8FAAB0BC16pY0QAAAELWZAGyEAAJtO0FoD2gcBAGC5CFrMjDZCAAA2laC1JpZp3PsoYQsAgE0kaAEAAPRM0Fozy7hfy3AMAAA2jaC1hpa1jRAAADaFoMXc2K8FAMCmELSYK2ELAIBNIGitsWXcrwUAAJtA0GLuDMcAAGDdCVprbpmrWtoIAQBYV4IWC6W6BQDAOhK0AAAAeiZobYBVuK+WNkIAANaJoMXSELYAAFgXgtYGWebBGAAAsE4ELZaK4RgAAKwDQWvDrEpVSxshAACrTNACAADomaDF0tJGCADAqhK0NtAqjHsfpY0QAIBVI2ixEoQtAABWiaDFytBKCADAqhC0NtiqTCDcTnULAIBlJ2ixkoQtAACWmaC14VZtMAYAAKwCQYskq9lGaM8WAADLStACAADomaDFjVaxqpXYrwUAwPKp1tqi17B0qupwkq2tra0cPnx40ctZiGtPfjFnX3jJopexZ5dfdE7OOHhg0csAAGBNnDhxIkeOHEmSI621E5O+TkWLtaK6BQDAMhC0GGtV2wgBAGAZCFqsHVUtAAAWzR6tMezRurlV3a+V2LMFAMD+TLtHy99AWWujAVHoAgBgXrQOLrOTn0uOHekeJz+3sGXYrwUAAHsjaLEx7N0CAGBeBC0msi5VLWELAIB5ELTYOGdfeEnOevrFAhcAADMjaDGxMw4eyJXPOXctKlsAADBLgtaqOH504UMxhrQRAgDA7sy6ZqMZ/w4AwCyoaDGVdalqAQDALAhaq+b40aVoH0zWb8+WVkIAAPoiaMEIEwkBAOiDoMW+rWMboeoWAAD7IWitoiWaQDi0bm2EibAFAMD0BC3YhVZCAACmIWjRq3VsI0xUtwAA2BtBi96tc9g66+kXq3ABAHBKgtYqW6JR79ut454tAACYlKAFU9BKCADAbgQtZmpd2wgTgzIAANiZoLXqlrh9cGidw1aiugUAwC0JWtADYQsAgFHVWlv0GpZOVR1OsrW1tZXDhw8vejldxer40Z2fv+Dq5OCh+a1nH649+cWcfeEli17GTF1+0Tk54+CBRS8DAIAenDhxIkeOHEmSI621E5O+TtAaY+WC1tCKBK5NCFtDQhcAwGqbNmhpHWTuNmn0u5ZCAIDNJGitgoOHumrVqazAYIxR6z4kY0jYAgDYPIIWC7VJYcsoeACAzSFosXBaCQEAWDeC1qo4eCg5tjVZCyFLTdgCAFh/gta6WbF9WqM2pY0QAID1J2ixVDaljdCeLQCA9SZoraPjR5NjR1a2srVJtBECAKwnNyweY+luWLzdmt3AeCebdGPjITc4BgBYLm5YzC2t8H6tTaXCBQCwHgQtltamDsewfwsAYPUJWqvo4KGNGfO+KcMxxlHdAgBYXYLWujMYY6UJWwAAq0nQYiVsahthclMroXZCAIDVYbzZqjp4KDm2NfkEQtbC6BRGEwoBAJaXihYrY5OrWuNoKwQAWF6C1qqbdDDGmox63+ThGAAArA59R7DCtBICACwnfyuDNSF0AQAsD62D62DD2gcT+7VOxf4tAIDF8k/erKzhfq1rT37xZtUcOtu/JqpcAADzo6LFylPdmszwflwqXQAAs1ettUWvYelU1eEkW1tbWzl8+PCil7M3k95X64Kru5bDNaKyNTnVLQCAyZw4cSJHjhxJkiOttROTvk5FCzaQ6hYAwGwJWqwN99jaO0MzAABmQ9BaNxs4gXA7e7b2RtgCAOifTRqsJRMJ98Y9uAAA+qWiBdyMChcAwP6ZOjjGSk8dHDXJBMI1nD64narW9FS3AIBNZ+og0DvVLQCA6ahojbE2Fa1ko++rNUpVa/9UtwCATaSiBbsw+n3/VLcAACbnn6eBiZlOCAAwGRWtdTfpfbU2hHtsAQAwD/ZojbFWe7QS+7TGsGerX6pbAMC6skcL9sCerX6dfeElOevpF9vDBQAwIGhxk+NHk2NHugoYTEHgAgDoaB0cY+1aB4e0EO5KO2H/tBQCAKtu2tZBfwOCgeGgDGGrP+O+lsIXALAJtA5ukkknEB4/urHtg6YSzp72QgBgEwhawEIIXADAOrNHa4y13aOVTL5PK9nYvVpD9mwthtZCAGCZTLtHS9AaY62DVmIoxh4IW4sldAEAi+Y+WvTPuHcWbNhe+MnPXpeznn6xVkMAYGWoaI2x9hWtIZWtPVHdWj4qXgDArKloMTsbPIVwlImEy+fsCy9R4QIAlpJ/CoY9OOPggVz5nHNVt5bI9u+DKhcAsAy0Do6xMa2DifbBfRC2VofwBQBMy9TBHglauxC4bkbYWk2CFwAwKXu0YAGGrYT2bq0We7sAgFlT0RpjoypaQ25kvG+qW6tJdQsA2M20FS1/u4CeGJSxmgzTAABmQUVrjI2saCUGY/RI2FofghcAbDbDMHokaE1A2DolYWs9CV4AsFkErR5tbNBKTCGcAYFrfQldALD+7NGCJXXGwQO5/KJzhK01ZH8XALAT491hDoZhi/V29oWX5KynX5yznn6x8fEAsOG0Do6x0a2DQ/ZrzZR2ws2hygUAq03rIKwQo+A3x+j3V+gCgM2hdZDxDh7qKlWTOH60q4CxZ1oKN8uwtVBbIQCsP/+0CgtmWMbmMUQDANafihY7O3goObY1WWXr+NHk2BGVLZjC2RdeosoFAGvGP6HCErBnC3u5AGC9mDo4hqmDY0w6hdAEwl4IXIwSvABgcaadOihojSFojWHc+0IIXOxE+AKA+Vjp8e5V9aQkP57kq5L8RZKntNbeuMv5t0nyrCTfneS2ST6Y5Mdaa78/7TVhGRmUwU52+pkQwABgOSy8olVVj07yy0melOTSJP8qyROSnN1a+/CY8w8Ozvt4kuNJrkpy5ySfaa29a5prjvkcKlrj7KWqlahs9Ux1i2kIXgCwPyvbOlhVf5rk7a21J44cuyLJK1tr5485/4fTVaq+rrX2hT6uOeb1gtY4gtbCCVv0RQADgMmsZNAaVKeuTfJPW2u/O3L8BUnu2Vp78JjX/H6STw1e951JPpHkvyX56dbaDVNe81ZJbjVy6NZJrhK0ttlr0EqErRkQtpg1IQwAbrKqe7TukOT0JNdsO35NkjN3eM3dkjw0ya8meWSSr0ny4nTv5aIpr3l+kp/Y49o3z/C+WtMELnpjFDyz5obKALB/y/J/zu1ltRpzbOi0dPuzfqi1dkOSt1XV0XTthBdNec1nJ3nuyMe3Trf3i/06flRVa0aGgStR5WK2BC8A2LtF/5/yk0luyC0rTXfMLStSQx9L8oVByBq6IsmZg7bBPV+ztXZ9kuuHH1fVpOuHpWA6IfPk5soAcGqnLfKTt9ZOJnlbkodve+rhSd60w8suTXL3qhpd+9cm+Vhr7eSU12QvDh7qqlSTOn40OXakazkE1srZF16Sa09+cdHLAIClswz/DPncJL9cVW9NclmSH0pylyQvSZKqekWSj45MC/y5JE9O8oKq+pl0e7QuSPLCSa8J60hVi0VxTy8AuKWFj3dPbry58NPS3Vz4PUn+bWvtfw2ee0OSK1tr542c/4Akz0tyzyQfTfILGUwdnOSaE6zHePdJmEK4tOzZYpkJYACskpUc776sBK0JCVpLT+BiVQhfACwrQatHgtYeuZHxShC6WDXCFwDLYFXvowXMiftvsWpMNwRglalojaGiNQVthCtF2GIdCF8AzIOKFjAxNztmHZh2CMAyU9EaQ0VrCtNUtBJVrSUhbLHuhC8ApqWixWIdPJQc25o+cAHMkOoXAPOmojWGitY+qGytLFUtuIkABsCQ8e49ErT2yWCMlSd0we4EMYDNIWj1SNDaJ1WttSFwwWQEL4D1ZY8W0LszDh7I5RedI2zBKZzq98hbn/Gw3PeZr7/xY8EMYP2paI2hotUDVa21pMIFsyWAASwfFS1g5lS4YLbG/d4SvgBWk4rWGCpaPVLZWluqW7A8hDGA2VHRAuZKdQuWx/bfh4Lj3dAuAAAgAElEQVQXwOKpaI2hotUzVa21prIFq0sgAzi1aStap81uScAmOOPggVz5nHNz5XPOzeUXnbPo5QB7cPaFl+Tak19c9DIA1pKK1hgqWjMwbVUrUdlaUSpdsB5UvYBNZ48WsFSGla5E6IJVZv8XwHRUtMZQ0Zoh+7U2msAF60fwAtadihaw9IZVLoEL1sduv5ff+oyH5b7PfP3Y5wQ0YN2paI2hojVjqloMCFzAOEIYsExUtICVo8IFjLPTnwcCGLBKVLTGUNGak71WtlS0NoLQBexG2ALmbdqKlqA1hqA1J1oImYDgBexG8AJmTesgsJbOOHggl190jrAFjLXbnw1CGLBIKlpjqGjN0TRVLRWtjaW6BeyH4AVMQ+tgjwStOdI+yB4JW8A87BbKtv85JMDBehO0eiRozZmwxR4JW8CqEMJg9QlaPRK0FsQUQqYgdAHrZtyNngU2WBxBq0eC1oIIWuyT0AVsitEwJoTBbAlaPRK0FkQLIT0SugCEMOiDoNUjQWuBhC16JGwB7E4Qg1NzHy2Abc44eCBXPufcJEIXwDh7+XNRKIO9UdEaQ0VrCdivxQwIWwDzJZyxDrQO9kjQWgJaCJkxoQtg8QQxVoGg1SNBa0lME7YELaYgdAGsFwGOPglaPRK0loSqFnMkbAGsv2EA2+nPfAGNcQStHglaS0RViwUTwAA2y/Z7lCU3DQ0RxDaToNUjQWuJqGqxJAQuAPZCKFsfxruzng4eSo5tTR+4AAAWYPQf50ZD117+4U5YW20qWmOoaC0hlS2WjAoXAPMmeC2G1sEeCVpLSthiCQlcACwDIWx2tA4CLMAZBw/kyuecu+PzghgA87CX/9cIZfOhojWGitYSM4WQFSVwAbBqBLKO1sEeCVpLbD9DMQQuloDABcA62kso2/7/wmUPdIJWjwStFaCyxRoQugBgd8sQwqYNWqfNbkmwZI4f7QIaLIkzDh648WaYAMB6Wd4aHcAGGDdMQ6ULAFaf1sExtA6uCOPe2RCCFwCbSusgADOjxRAAVo+gxeo6eCg5ttVVqPbi+NHk2BH7tVgpwhYArBZ7tNhcx49qI2Sl7HRzZK2FALB8BC1W38FDXWCa9v5asOJGA5jQBQDLQdBis6lqsWaELgBYDoIW60FVC25he6uh4AUA8yNoAWyInfZ4JUIYAPTNfbTGcB+tFef+WrBvghcAy2CV76OlogVD9mvBjbQdAsD+CFqsH/u1oHeGbADA3rhhMYxyM2M4pWHocgNlANiZoMV6OngoObbVVbaAmRC4AGBnhmGMYRjGmjEcAxZOuyEA01jlYRgqWgDM3LD6tb0C9tZnPGyBqwKA2VHRGkNFaw2pasHKUQUDYJUrWqYOArCUTDoEYJVpHYTdmEIIS+GMgwcM3QBgpahosRmGUwinbSEEFm77TZSHVLsAWEb2aI1hj9aas18LNoIABrD67NECgCWjAgbAIglaMKnjR1W1YA0YsgHAPGgdHEPr4AbY714tgQvWkuAFsFy0DsKqMRwDGEO1C4C+CFowDW2EsPZ22uM1ShgDmK0v3vClRS9haoIWm+3goS4wqWoBUzBwA2C2Tj+tFr2EqQlaMK1hOFPZArYZF8CEL4C9q1rdoGUYxhiGYWyo/ezXEraAPRC6ACZjGAasA22EwJxsr3gJXgDrR9ACgAWbZPDGJAQ2gOWhdXAMrYMbbtoWQu2DwAoRyoBVoHUQMBwDWCnG1wPMlorWGCpaJDEcA2AbwQuYNxUtAGDt7WcvmZAGbBoVrTFUtLjRfqpaicoWwD4JaLDZVLQAAGZAFQ1YVSpaY6hocTOqWgBrQ/iC1aKiBezs+FFhC2BJjFbIhC5glgQtOJWDh5JjW/uvbAGwVIy4B2ZJ6+AYWgfZkTZCgI223+D11mc8LPd95ut7XBGsN62DAAAbYD/DOYa0LsJmUNEaQ0WLU3IzYwDmSChjU6loAZMzHAOAPeqjkrad8AazJWjBNA4e6sKS4RgArKjt4U3wgn5pHRxD6yATMxwDAJIIasyG1kEAADbapO2NAhmbQkVrDBUt9sxwDADo1WggMxZ/c61yRUvQGkPQYiraCAFgISatkglsq2eVg5bWQQAAVtpepjLuZXrjpFU1AY5xVLTGUNFiX7QRAsBGGw1ol190TpLYlzYlFS3gJka/A8BGG1dh2+ugkJ0ChmEiq0PQgmXiZsYAsLEmaYHc7ZzdQpj2xvnTOjiG1kF6YTgGALAilrVSpnUQAABYWXsZKLKTvYzk34QKm4rWGCpa9MpwDACAlTVtReu02S0JAABgMwlaMGvDKYTTOH60q4gBALBSBC1YdsePJseOCFwAACtE0IJ52E9VCwCAlSNowbwcPJQc29JGCACwAQQtAACAnglaMG/7HY5hvxYAwNITtGAVaSMEAFhqghYswn73awEAsNQELVgk99gCAFhLghYsmj1bAABrR9ACAADomaAFy8A9tgAA1oqgBctEGyEAwFoQtAAAAHomaMGy2U9VK9FGCACwBAQtWEfCFgDAQlVrbX8XqDqc5KFJ3tdau6KXVS3Y4D1tbW1t5fDhw4teDpvs5Oe60LQfF1zdVckAANizEydO5MiRI0lypLV2YtLX7bmiVVW/WVX/ZvDrL0/y1iS/meTPq+qf7PV6wC7220aYqG4BACzANK2DD0ryxsGvvytJJblNkh9J8oye1gUAALCypglaR5J8avDrRyT57dbatUkuTvI1fS0MGNjvPbYSVS0AgDmbJmh9JMkDqupQuqD12sHx2ya5rq+FAdv0MY3QfbYAAOZimqD1/CS/muSqJFcnecPg+IOSvLufZQEzo7oFADBzU00drKr7Jrlzkte11j47OHZukk+31i7td4nzZ+ogS62PSYRDJhICAOxq2qmDB6b5ZK21t6abNpiqOj3J30vyptba30xzPWAPhnu2+gxcAAD0aprx7s+vqh8Y/Pr0JP8zyduTfKSqHtLv8oCZ0kYIADAT0+zR+p4k7xr8+lFJ/p8kX5du79azeloXcCp93GMrEbYAAGZgmqB1hyT/Z/DrRyb576219yf5hXQthMC89DH6HQCA3k0TtK5JcvagbfARSf5wcPyMJDf0tTBgjox+BwDo1TRB62VJfjPJe5K0JK8bHP8HSd7b07qAvdBGCACwVPYctFprx5I8IclLk/zD1tr1g6duSPKc/pYG7ImwBQCwNKa6j9a6cx8tVl5fo9/dZwsA2HDT3kdrmtbBVNWDq+pVVfWXVfWBqvofVfUt01wLAABg3UxzH63HpRuAcW2SFyZ5UZLPJ3l9VT223+UBU9FGCACwUHtuHayqK5K8tLX2vG3Hn5rkB1trX9/j+hZC6yBro68WwkQbIQCwkebZOni3JK8ac/x/pLt5MbAs3GcLAGAhpglaH0nysDHHHzZ4Dlg2fbQSaiMEAJjYgSle81+SvLCq7pnkTenupfXNSc5L8qP9LQ1YOsePaiEEAJjAVOPdq+q7kvxYkuF+rCuS/KfW2u/1uLaFsUeLtWb0OwDAxKbdo9XbfbSq6m8l+arW2od7ueACCVqsPWELAGAic72P1g7OTvLBHq8HAACwkvoMWsCq6PM+W8eOGJIBALCNoAWbqs/R7yYSAgDcjKAFAADQs4nHu1fVN57ilHvscy3AIgzbCPc7HMPodwCAG008dbCqvpTunlk15unh8dZaO72/5S2GqYNspL4mESYCFwCwNmY+3r2q7jrJea21D036yZeVoMVGM/odAOBG0watiVsH1yFAAXOklRAA2GCGYQA319fodwCADSZoAbPjPlsAwIZaeNCqqidV1Qer6rqqeltVfcsu555XVW3M48tGzjlQVc8cXPPzVfW/q+rCqlr4e4WV0ec9thL32QIANs5Cw0dVPTrJ85M8K8m9krwxyWuq6i67vOxEkq8afbTWrht5/v9L8sNJ/k2Sr0/ytCQ/nuTJvb8BWHd9thGqbgEAG2TiYRgz8tQkv9Ba+/nBx0+pqnOSPDHJ+Tu8prXW/s8u13xAkt9rrV08+PjKqnpMkvv2smIAAIBT2HNFq6reUVVvH/N4W1VdWlW/VFXfOsF1Dia5T5LXbnvqtUkeuMtLv6KqPlRVV1XVq6vqXtue/5MkD6uqrx18nm9K8s1Jfn+Xtdyqqg4PH0lufar1w8boeziGNkIAYANM0zr4B0nuluRzSf44yRuSfDbJ30nyZ+na+f6wqr7zFNe5Q5LTk1yz7fg1Sc7c4TXvTXJeku9I8pgk1yW5tKq+ZuScn07ya0neW1VfSPKOJM9vrf3aLms5P8nWyOOqU6wdNos9WwAAezJN0LpDkv/SWvuW1tqPtdae2lp7UJL/nORQa+0fJXlmkv8w4fW23zG5xhzrTmztza21X2mtvau19sYk/yzJ+3Pz/VePTvK4JI9Ncu8kj0/y76rq8bus4dlJjow87jTh2mGz2LMFADCRafZo/bN0LX/b/XqStyX5wXQVpaee4jqfTHJDblm9umNuWeUaq7X2par6sySjFa3/lOQ5rbVfH3z87qq6a7qq1S/tcJ3rk1w//LiqJvn0AAAAY01T0bou4/dQPXDw3PC6148550attZPpgtnDtz318CRvmmQh1SWieyb52MjhM5J8adupN2QJRtnDWtBGCABwStNUtH4myUuq6j7p9mS1JH8/yROSHB+cc066vVGn8twkv1xVb01yWZIfSnKXJC9Jkqp6RZKPttbOH3z8E0nenOQDSQ4n+ZF0Qetfj1zzVUn+fVV9OMlfpBsb/9QkvzjFewV2MmwjPH50/9c6frS71sFD+78WAMAS2HPQaq09s6o+mO4+Vf9icPh9SX6wtfbfBh+/JMnPTXCt36iq2ye5MN0QjfckeWRr7UODU+6Sm1enbpPkpenaDbfShbkHtdbeMnLOk5P8VJKfTdeGeHWS/5rkoj2+VWCehoFN4AIA1kC1NnbuxEYbjHjf2trayuHDhxe9HFhuJz/XT1VrlLAFACyJEydO5MiRI0lypLV2YtLXTX3D4kHr4Nenax28vLU2SasgsG6Ge7ZmEbgAAFbUnitaVXXHdBMGH5Lk0+nGsR9Jd0+t722tfaLnNc6dihZMSXULAFgz01a0ppnE9zPpBlH83dba7Vprt03yDYNjL5ziegAAAGtlmqD1iCRPbK1dMTzQWrs83eS/b+trYcAK6vOGxkPGvwMAK2iaoHVaki+MOf6FKa8HrJO+77OVdGHr2BGBCwBYGdMEoz9K8oKqunEjRlV9dZLnJXl9XwsDVpzqFgCwwaYJWv8mya2TXFlVf1VVf5nkg4NjP9Ln4oAVN4vqFgDACpj6PlpV9fAkX5du6uDlrbU/7HNhi2TqIMxA3xMJTSMEAOZgnlMHkySttde11n6mtfbC1tofVtWdq+oXp70ewJ5oIwQAllifwytul+TxPV4PWCf2bAEAG8SUQGB+TCQEADaEoAWsB9UtAGCJCFrA/M2ijRAAYIkcmPTEqvqdU5xym32uBdgkw7DV5yTC40dNIwQAlsLEQSvJ1gTPv2IfawE2zXDPVp+j34fXEbgAgAWa+j5a68x9tGAB+r7PViJsAQD7Nvf7aAEsPRMJAYAFEbSA5TCL0e8AAAsiaAHLxY2NAYA1IGgBy0fYAgBWnKAFbA57tgCAORG0gOU0yz1bqlsAwIwJWsBym0UbYaK6BQDMlKAFLD8TCQGAFSNoAavDkAwAYEUIWgDCFgDQM0ELWC32bAEAK0DQAlaPiYQAwJITtIDVNcvqlrAFAOyDoAUAANAzQQtYbfZsAQBLSNACVp89WwDAkhG0gPWhugUALAlBC1gvs6xuAQBMSNAC1tMsqlvaCAGACQlawPoStgCABanW2qLXsHSq6nCSra2trRw+fHjRywH6cPJzXUjq0wVXd2EOAFhbJ06cyJEjR5LkSGvtxKSvU9ECNsOsBmUAAIwhaAFMSxshALADQQvYHLOYSChsAQBjCFrA5um7jdB9tgCAbQQtYDOZSAgAzJCgBdAn1S0AIIIWsMlmsWdrSHULADaaoAUwq9HvwhYAbCxBCyCZbdjSSggAG0fQAhjSSggA9ETQAthOdQsA2CdBC2Ac1S0AYB8ELYDdGJQBAExB0AI4lVmFLQBgbVVrbdFrWDpVdTjJ1tbWVg4fPrzo5QDL5OTnumpUny64ugtzAMDSOXHiRI4cOZIkR1prJyZ9nYoWwF7MorqljRAA1o6gBbAMhC0AWCuCFsBeGf8OAJyCoAUwDePfAYBdCFoA+6G6BQCMIWgB7JfqFgCwjaAF0Bc3NwYABgQtgD65uTEAEDcsHssNi4FeuLkxAKw8NywG2ATaCAFgJQhaALNizxYAbCytg2NoHQR6N4s2wkQrIQDM2LStg4LWGIIWMBOzCltDQhcA9M4eLYBlZyIhAGwMQQtgntzcGAA2gqAFsAgGZQDAWhO0ABZllmHr2BGBCwAWSNACWKRZthICAAsjaAGsK22EALAwxruPYbw7sFCzGANv9DsATMV4dwB2proFAHMlaAEsGxMJAWDlCVoAy8hEQgBYaYIWwLJyc2MAWFmCFsCy00oIACtH0AJYBVoJAWClCFoAqG4BQM8ELYBVMcs9WwBArwQtgFWjjRAAlp6gBbCKTCQEgKUmaAGsMhMJAWApCVoAq04rIQAsnWqtLXoNS6eqDifZ2trayuHDhxe9HIDJnfxcF5Bm4YKru1AHABvkxIkTOXLkSJIcaa2dmPR1KloA62RW1a1EhQsA9kDQAlg3xsADwMIJWgDryqAMAFgYQQtgnRmUAQALIWgBrDv33AKAuRO0ADaFVkIAmBtBC2CTaCUEgLlwH60x3EcL2BizuO+W+20BsEamvY+WoDWGoAVsHIELAMZyw2IAlou9WwBsMEELAIMyAKBnghYAHYMyAKA3ghYAN3HPLQDohWEYYxiGAZDZDMgYZVgGACvA1MEeCVoA28wqdAlbACw5UwcBWD32bwGwpgQtAE5tVoMyhuzfAmDNCFoATGaWgzISYQuAtWKP1hj2aAFMYJbDMuzdAmBJ2KMFwHwZBQ8AOxK0AFhOwhYAK0zQAmB/Zjkow1RCAFaUPVpj2KMFMKVZ3+Q4sX8LgLmyRwuAxZv1GHgAWBGCFgD9msc9t7QTArDktA6OoXUQoCdaCQFYcVoHAVg+82glNJ0QgCUkaAEwW/MKW9oJAVgighYAszfLmxuPUt0CYEkIWgDMj1ZCADaEYRhjGIYBMAfzGJSRGJYBwL4YhgHAapnXPbdUuABYAEELgMWZZ9gyLAOAORK0AFiseQ3KSFS3AJgbe7TGsEcLYIHs3QJgidijBcB6MAoegDUgaAGwnIyCB2CFaR0cQ+sgwJKZRzuhVkIAxtA6CMD6Ut0CYMUIWgCshnmFLWPgAeiBoAXA6jAoA4AVYY/WGPZoAayQWe7fsm8LYONNu0frwOyWBAArbjTACV0A7IHWQQBWm3ZCAJaQ1sExtA4CrDCj4AHokfHuAJAYBQ/AUhC0AFg/RsEDsGBaB8fQOgiwRubRSphoJwRYU1oHAWCceVS3EhUuAG5G0AJg/c1rMmFi/xYASQQtADbJPKtbwhbARrNHawx7tAA2hFHwAJyCPVoAsIxUtwA2kqAFwOYyKAOAGdE6OIbWQYANNK8x8Il2QoAVonUQAPZjXtWtRIULYAMIWgAwNM+wldi/BbDGtA6OoXUQgLm2EibaCQGWlNZBAOiT6hYA+yBoAcBOhC0ApqR1cAytgwDcglZCgI00beugoDWGoAXAroyCB9gY9mgBwLwYBQ/AKQhaADAN+7cA2IXWwTG0DgKwJ/ZvAawtrYMAsCiqWwBso6I1hooWAFOZd2VrSIULYGZUtABg0Q4eSo5tzbe6lahwASyhhQetqnpSVX2wqq6rqrdV1bfscu55VdXGPL5s23lfXVW/UlV/XVXXVtU7q+o+s383AJD5txImwhbAkjmwyE9eVY9O8vwkT0pyaZJ/leQ1VXV2a+3DO7zsRJJ7jB5orV03cs3bDq71x0m+LcnHk/ydJJ/u/Q0AwE6G1a15thOOfh7thAALtdA9WlX1p0ne3lp74sixK5K8srV2/pjzz0vy/NbabXa55nOS/MPW2o6VsTGvuVWSW40cunWSq+zRAqAXi9q7lQhcAPu0cnu0qupgkvskee22p16b5IG7vPQrqupDVXVVVb26qu617fnvSPLWqvrvVfXxqnpHVf3gKZZzfpKtkcdVk78TADiFRbQSDmkpBFiIRe7RukOS05Ncs+34NUnO3OE1701yXrow9Zgk1yW5tKq+ZuScuyV5YpIPJDknyUuSvLCq/uUua3l2kiMjjzvt5Y0AwCkNWwkNywDYCAvdozWwvXexxhzrTmztzUnefOOJVZcmeXuSJyf5kcHh05K8tbV2weDjd1TV300Xvl6xw3WvT3L9yHX3/i4AYFKL3L+llRBgLhZZ0fpkkhtyy+rVHXPLKtdYrbUvJfmzJKMVrY8luXzbqVckuct0ywSAGVnUdMJjR7qHKhfAzCwsaLXWTiZ5W5KHb3vq4UneNMk1qis93TNduBq6NNumEib52iQfmm6lADBD9m8BrKVF30fruUmeUFXfX1VfX1XPS1d5ekmSVNUrqurZw5Or6ieq6pyqultV3TPJL6QLWi8Zuebzkty/qi6oqrtX1WOT/FCSF8/rTQHAnizqRseJsAUwIwvdo9Va+42qun2SC5N8VZL3JHlka21YfbpLki+NvOQ2SV6art1wK8k7kjyotfaWkWv+WVV9V7oBFxcm+WCSp7TWfnXW7wcA9mURe7cS+7cAZmCh99FaVlV1OMmW+2gBsFCLuv+WwAVwo5W7jxYAcAqL2r81HJihpRBgaoIWACwz+7cAVpKgBQCrQHULYKXYozWGPVoALD37twDmwh4tANgkKlwAS03QAoBV5WbHAEtL0AKAVWZYBsBSskdrDHu0AFhJi9q3NWT/FrCGpt2jJWiNIWgBsPIWGboELmCNGIYBANxk0fu3DMwANpyK1hgqWgCsHRUugKmoaAEAOzM0A2CuBC0A2CTuvwUwF4IWAGwa+7cAZs4erTHs0QJgYxgJD7Are7QAgL1bZHUrsX8LWFsqWmOoaAGwsRZV4VLZApaUGxb3SNACYKNpJwS4kaDVI0ELACJwAcQeLQCgb4u891Zi/xaw0gQtAGB3bnYMsGeCFgAwGTc7BpiYoAUATM7NjgEmYhjGGIZhAMAEDMsANoCpgz0StABgj4QuYE2ZOggALM4iWwoTQzOApSNoAQD9ELYAbqR1cAytgwCwT4tuJUy0EwK9sEerR4IWAPRo0aFL4AL2wR4tAGA5aSkENpCK1hgqWgAwQypcwApR0QIAmIQKFzAHKlpjqGgBwBwsurI1pMIF7MIwjB4JWgAwZ4sOXcIWsANBq0eCFgAsyKIDVyJ0ATdjjxb83/buPViSq74P+Pdn4RVYj8WAsRAgwDbIxhBELQRMeNlCYKiUi0diq+KUUSqFKRyCASfGIkasQyLbwSYCRIKJHRQwBEhSmAhCwILIiSXhIAmQhMCSXYhY6GVLsCsJ9OTkj+6RZkezu3d3+97pmfl8qrrune7TM917T+2d7/2dcwaA5bfoFQoTc7iAQahozaGiBQALNobK1oQKF6w1QwcHJGgBwEgIXMCCGToIAKyeMQwlnDj92GTndsMKgQ0RtACAcRtT2EoELmBDDB2cw9BBABixMQ0nTAwphBVnjtaABC0AWAICF7AFzNECANbLGIcUGk4I9FS05lDRAoAlpMIFbAIVLQBgvalwASOiojWHihYALDnVLWAgFsMYkKAFACtC4AIOkaA1IEELAFaMwAUcJHO0AAD2ZtsRyc5d45nDZf4WrDwVrTlUtABgxY2pwqW6BaOmogUAsFFjWqHw9GOTndtVuGDFqGjNoaIFAGtkTNWtRIULRsZiGAMStABgDY0tcCVCF4yAoDUgQQsA1pjABUwxRwsAYAhjmr81YR4XLB0VrTlUtACAJOOsbiUqXLCFDB0ckKAFAOxB4IK1ZeggAMBmGdsHHk/44GMYLRWtOVS0AIB9UuGCtaGiBQCwVVS4gP1Q0ZpDRQsAOGBjq3KpbsEgVLQAABZpbMvCq27BQqlozaGiBQAcMhUuWAkqWgAAY6LCBWtNRWsOFS0AYDBjq2xNqHDBhqhoAQCMkRUKYS2paM2hogUAbKoxVrlUuGAuFS0AgGUxtvlbSRf8dm5X5YKBqGjNoaIFAGypsVW4VLfgHgdb0RK05hC0AIAtN7awNSF0seYMHQQAWGYWzYCVoqI1h4oWALBQY61uJSpcrB1DBwckaAEAozHG0CVssUYErQEJWgDAqIwxbE0IXaw4QWtAghYAMDpjDlsTQhcrSNAakKAFAIzamEOXsMWKseogAMC6GOsKhYlVCqGnojWHihYAsFRUuGDTGDo4IEELAFhKAhcMztBBAIB1Z0ghjIagBQCwasYauIQt1oihg3MYOggArAzDCeGQmKM1IEELAFhJYw1dAhcjZo4WAAD7tu2I8Q0nTLrwt3O7YYWsFBWtOVS0AIC1oMIF+2Xo4IAELQBg7YwxdAlcjIChgwAAHLwxDis0pJAlpqI1h4oWALDWxljdmlDlYoupaAEAMIyxfg5X4rO4WBoqWnOoaAEATFHhYo1ZDGNAghYAwF6MMXQJW2wiQwcBANh8Fs2ADRG0AAA4MGMMW4nAxagYOjiHoYMAAAdgjMMJJwwr5BAZOggAwGKMtcKVWKWQhVHRmkNFCwDgEKhwsUKsOjggQQsAYAACFyvA0EEAAMbFBx+zxgQtAAA211gDl7DFJjJ0cA5DBwEANtkYhxUaTsgc5mgNSNACANgCYwxbE0IXPXO0AABYLpaFZ4WpaM2hogUAsABjrXCpbq01FS0AAJbbWCtcqlscBBWtOVS0AABGQIWLEVDRAgBgtVgWniWmojWHihYAwAipcLEAKloAAKw2FS6WiIrWHCpaAABLYlOW88wAABFmSURBVGxVLtWtlaOiBQDA+hnbSoWnH5vs3K7ChYrWPCpaAABLSoWLgR1sRUvQmkPQAgBYcmMLXInQtaQMHQQAgIkxLpxh0Yy1oqI1h4oWAMAKGluVS4VrKahoAQDAvoytyqXCtdJUtOZQ0QIAWAMqXGyAxTAGJGgBAKyRsQWuROgaEUFrQIIWAMAaEriYQ9AakKAFALDmxha6BK6FEbQGJGgBADC6sDUhdG0pQWtAghYAAHsYY+gSuLaE5d0BAGCzbDtiPMvCT5x+bLJzuyXiR0rQAgCAjRjb53BN+DyuUTJ0cA5DBwEA2K8xDidMDCkcmDlaAxK0AADYsLEGrkToGoCgNSBBCwCAAyZwrSRBa0CCFgAAh2SsoUvgOmBWHQQAgLGwcMbaU9GaQ0ULAIBBqXAtLRUtAAAYKxWutaOiNYeKFgAAm2qMFS7VrbkshjEgQQsAgC0hcI2eoDUgQQsAgC0ndI2SOVoAALDMth0xzjlcO7ebx3UQVLTmUNECAGDhVLhGwdDBAQlaAACMwhjD1sSahC5Ba0CCFgAAoyN0LYQ5WgAAsMrGOIdrwudx3YeK1hwqWgAAjJ4K15YwdHBAghYAAEthzGFrYslDl6A1IEELAIClM/bQtaSBS9AakKAFAMDSGnvgSpYqdAlaAxK0AABYegLXIAStAQlaAACslDGHrpGHLUFrQIIWAAArZ8xha2KEoUvQGpCgBQDAShO6NswHFgMAABuz7Yhk567xfgBysvQfgixoAQDAulqGwLWk7rfoCwAAABZsEriS5RhWuARUtAAAgHupcg1C0AIAAO5L4Dokhg4CAAB7Z1jhQRlFRauqfqmqvlZVt1XVRVX1rH20PaWq2pzt/ntpf2p//IzNuwMAAFgD245Q4dqghVe0qurnkpyR5JeSnJfklUk+WVWPb639v72ctjvJ8dM7Wmu3zXnupyb5xSSXDHrRAACwrraywnXnd0bxWVoHYwwVrdcn+YPW2u+31r7SWnttkr9K8qp9nNNaa9dNb7MNqurIJB9I8ook39yUKwcAgHU2CV2bNZfrzu8M/5xbZKFBq6q2JdmR5NMzhz6d5Bn7OPXIqvp6VV1dVR+vqifPafOuJJ9orZ2zges4vKqOnmxJjtroPQAAANmcYYXf96Bhn28LLXro4EOSHJbk+pn91yc5Zi/nfDXJKUkuTXJ0kl9Ocl5VPam1dmWSVNXJ6QLcUzZ4HacmefMBXTkAALAnC2fcY9FBa6LNPK45+7qGrX0uyefuaVh1XpKLk/zTJK+pqkcmeXuS58+bt7UXv5nkbVOPj0py9QbPBQAAZq156Fr0HK2/SXJ37lu9emjuW+Waq7X23SSfT/LYfteO/vyLququqroryXPShbC7quqwOc9xe2tt92RLcvPB3Q4AAHAfa7ha4UIrWq21O6rqoiQnJfno1KGTknxsI89RVZXkhHRDCZPkM0meONPsvemGHP52a+3uQ7poAADgwK1ZhWsMQwffluT9VXVhkgvSLcd+XJJ3J0lVvS/JN1prp/aP35xu6OCV6eZovSZd0PonSdJauznJZdMvUFW3JrmxtbbHfgAAYAEmFa4VDlsLD1qttQ9X1YOTnJbkYelC0otaa1/vmxyX5LtTpzwwyXvSDTfcleQLSZ7dWvu/W3fVAADAIZlUuFa0ulWtzV1zYq31S7zv2rVrV44++uhFXw4AAKyH2dD1xmsW/oHFu3fvzvbt25Nke7+ew4YsvKIFAACQZM95XEtu0asOAgAArBxBCwAAYGCCFgAAwMAELQAAgIEJWgAAAAMTtAAAAAYmaAEAAAxM0AIAABiYoAUAADAwQQsAAGBgghYAAMDABC0AAICBCVoAAAADE7QAAAAGJmgBAAAMTNACAAAYmKAFAAAwMEELAABgYIIWAADAwAQtAACAgQlaAAAAAxO0AAAABiZoAQAADEzQAgAAGJigBQAAMDBBCwAAYGCCFgAAwMAELQAAgIEJWgAAAAMTtAAAAAYmaAEAAAxM0AIAABiYoAUAADCw+y36AsZs9+7di74EAABggQ42E1RrbeBLWX5V9fAkVy/6OgAAgNF4RGvtGxttLGjNUVWV5NgkNy/6WpIclS70PSLjuB7GS19hI/QTNkI/YSP0EzZiVfrJUUmuaQcQngwdnKP/B9xwWt1MXeZLktzcWjOWkb3SV9gI/YSN0E/YCP2EjVihfnLA124xDAAAgIEJWgAAAAMTtMbv9iS/0X+FfdFX2Aj9hI3QT9gI/YSNWNt+YjEMAACAgaloAQAADEzQAgAAGJigBQAAMDBBCwAAYGCC1oJU1bOr6uyquqaqWlW9eOZ4VdXO/vh3qurcqvrxmTbfX1Xvr6pd/fb+qnrg1t4Jm6WqTq2qz1fVzVV1Q1X9UVUdP9Pm8Kp6Z1X9TVXdWlX/vaoeMdPmuL6v3dq3e0dVbdvau2EzVdWrquqSqtrdbxdU1Qunjusn7KH//6VV1RlT+/QT0r/3aDPbdVPHvT8hSVJVD6+qP6yqG6vq21X1xaraMXV87fuKoLU4RyT5UpJX7+X4ryZ5fX/8qUmuS/LHVXXUVJsPJjkhyU/32wlJ3r9ZF8yWe06SdyV5epKTktwvyaer6oipNmckeUmSk5M8M8mRST5eVYclSf/1E+n62zP7di9L8rtbdA9sjauT/FqSp/TbZ5N8bOoXmn7CParqqUl+McklM4f0Eya+nORhU9sTp455f0Kq6vuTnJfkziQvTPL4JL+S5FtTzfSV1pptwVuSluTFU48rybVJ3jC17/B0nfeV/eMf68972lSbp/f7jl/0Pdk2pZ/8QP/zfXb/eHuSO5L83FSbY5PcneQF/eMX9o+PnWpzcpLbkhy96HuybWp/uSnJP9ZPbDP94sgkVyR5XpJzk5zR79dPbJOf6c4kX9zLMe9PbJOf6W8l+T/7OK6vtKaiNVKPSXJMkk9PdrTWbk/yJ0me0e/6iSS7Wmt/NtXmc0l2TbVhtWzvv97Uf92R5HuzZz+5Jsll2bOfXNbvn/hUuv/sdoSVU1WHVdXJ6aoOF0Q/YU/vSvKJ1to5M/v1E6Y9th/u9bWq+lBV/VC/3/sTJn4myYVV9V/66Q1fqKpXTB3XV2Lo4Fgd03+9fmb/9VPHjklyw5xzb5hqw4qoqkrytiR/2lq7rN99TJI7WmvfnGk+20/26Ed9+zuin6yUqnpiVd2S5PYk707yktba5dFP6PUBfEeSU+cc1k+Y+LMkv5DkBUleke5ne35VPTjen3CvH0ryqiRXpusr707yjqr6hf64vpJuzgfj1WYe18y+2ePz2rAazkzyt9LNi9gf/WQ9/Xm6se0PTDdv5j9V1XP20V4/WSNV9cgkb0/y/NbabQdyavSTtdJa++TUw0ur6oIkf5nk5Uk+N2k2c5p+sn6+J8mFrbU39o+/0M8LflWS9021W+u+oqI1TpPVfWbT/ENz718Grkvyg3PO/YHc968HLLGqeme6Ev1Pttaunjp0XZJt/YTUabP9ZI9+1Lf/3ugnK6W1dkdr7S9aaxe21k5Nt9jOL0c/obMj3c/8oqq6q6ruSrfgzmv676+PfsIcrbVbk1ya5LHx/oR7XZvk8pl9X0lyXP+9vhJBa6y+lq7znTTZ0S+f+5wk5/e7Lkiyvar+9lSbp6Wbx3N+WHr9sqhnJnlpkp9qrX1tpslF6Vb7me4nD0vyhOzZT57Q7594frrhZRdt1rUzCpVu7ox+QpJ8Jt3KcSdMbRcm+cDU9/oJ91FVh6dbtODaeH/Cvc5LcvzMvscl+Xr/vb6SWHVwUVu6lZ8mv+xaktf13x/XH39DupVZXpLuF90Hk1yT5Kip5/hkur9aP73fLkly9qLvzTZYH/l3fR94Trq/CE22B0y1+fdJ/irJiUmenO7N1BeTHNYfPyzdXyLP6Y+f2Ld/56LvzzZoXzk9ybOSPDrdm+l/nW51uJP0E9s++s256Vcd1E9sU/3gd/rfO49J8rQkZyfZneRR/XHvT2xJt1z7nUnemORHkvyDJLcm+fmpNmvfVxZ+Aeu6JXluuoA1u53VH690S6xem27p3D9J8oSZ53hQkj/s/wPc3X//wEXfm22wPjKvf7Qkp0y1uX+Sdya5Mcm3+1+Ij5x5nuOSfLw/fmPf/vBF359t0L7yB0muSldZuKF/I3ySfmLbT785N3sGLf3EliQf6t8M35HkG0n+W5LHTx33/sQ2+Tn/3XR/fLkt3bDBV8wcX/u+Uv1NAgAAMBBztAAAAAYmaAEAAAxM0AIAABiYoAUAADAwQQsAAGBgghYAAMDABC0AAICBCVoAAAADE7QAWGtVdVVVvXbR1wHAahG0AFgLVXVKVX1rzqGnJnnPFry+QAewRu636AsAgEVqrf31oq/hQFTVttbaHYu+DgD2TUULgC1VVedW1Tuq6t9U1U1VdV1V7dzgudur6j1VdUNV7a6qz1bVk6aOP6mq/ldV3dwfv6iqnlJVz03y3iTbq6r1287+nD0qTf2xV1bVx6vq21X1lar6iar6kf7ab62qC6rqh6fO+eGq+lhVXV9Vt1TV56vqedP3nORRSf7t5PWnjr2sqr5cVbf31/IrM/d8VVX9elWdVVW7kvyHqtpWVWdW1bVVdVvf5tQD+kEAsKkELQAW4eVJbk3ytCS/muS0qjppXydUVSX5RJJjkrwoyY4kFyf5TFU9qG/2gSRXpxsOuCPJbyW5M8n5SV6bZHeSh/Xb7+zj5d6U5H1JTkjy1SQfTPJ7SX4zyVP6NmdOtT8yyf9I8rwkT07yqSRnV9Vx/fGX9td12tTrp6p2JPlIkg8leWKSnUneUlWnzFzPP09yWX9Pb0nymiQ/k+Rnkxyf5B8muWof9wPAFjN0EIBFuKS19hv991dW1auTnJjkj/dxzk+mCyMPba3d3u/7Z1X14iR/L908q+OSvLW19tXJc09O7qtBrbV23Qau772ttY/05/12kguSvKW19ql+39vTVciS7km/lORLU+f/elW9JF0YOrO1dlNV3Z3k5pnXf32Sz7TW3tI/vqKqHp8uWJ011e6zrbV7gmEf4K5M8qettZbk6xu4JwC2kIoWAItwyczja5M8dD/n7EhXObqxH553S1XdkuQxSSbD+N6W5Per6pyq+rXp4X2HcH3X918vndl3/6o6Okmq6oh+KOTlVfWt/rp+NF3w25cfS3LezL7zkjy2qg6b2nfhTJuz0lXb/rwfhvn8/d4RAFtK0AJgEe6cedyy/99J35MukJ0wsx2f5K1J0lrbmeTH0w0x/Kkkl/eVpUO5vraPfZNrfmuSlyX5F0me1V/XpUm27ed1auq5pvfNunX6QWvt4nQB801JHpDkI1X1X/fzWgBsIUMHAVgWF6ebn3VXa+2qvTVqrV2R5Ip0C0/85yT/KMlHk9yR5LC9nXeInpXkrNbaR5Okqo5M8uiZNvNe//Ikz5zZ94wkV7TW7t7XC7bWdif5cJIP9yHrf1bVg1prNx3cLQAwJBUtAJbFOenmSv1RVb2gqh5dVc+oqn/Vryz4gH4lvudW1aOq6u+kWxTjK/35VyU5sqpOrKqHVNX3DXhtf5HkpVV1Qr8K4gdz39+xVyV5dlU9vKoe0u/73SQnVtWbqupxVfXyJK/OvhfqSFW9rqpOrqofrarHJfn7Sa5LMu9zwgBYAEELgKXQL/rwoiT/O8l/TFe1+lC6ytH1Se5O8uB0qwVekW41v08meXN//vlJ3p2uCvTX6VY7HMrrknwz3eqGZ6dbdfDimTan9df6l/3rT4YA/mySk9OtKvgvk5zWWjtrP693S5I3pJu79fn+eV/UWvvuId8JAIOo7vcWAAAAQ1HRAgAAGJigBcAoVNXPTy/bPrN9edHXBwAHwtBBAEahqo5K8oN7OXxna82H8gKwNAQtAACAgRk6CAAAMDBBCwAAYGCCFgAAwMAELQAAgIEJWgAAAAMTtAAAAAYmaAEAAAzs/wNouRlTRntHfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1000x1000 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "\n",
    "cvresult = cvresult.iloc[100:]\n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(100,cvresult.shape[0]+100)\n",
    "        \n",
    "fig = pyplot.figure(figsize=(10, 10), dpi=100)\n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators_detail.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二步：调整树的参数：max_depth & min_child_weight\n",
    "(粗调，参数的步长为2；下一步是在粗调最佳参数周围，将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一轮参数调整得到的n_estimators最优值（699），其余参数继续默认值\n",
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {}\n",
    "param['gpu_id'] = 1\n",
    "param['updater'] = 'grow_gpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59040, std: 0.00387, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59078, std: 0.00454, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.59049, std: 0.00446, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.59125, std: 0.00485, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.59115, std: 0.00444, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.59056, std: 0.00385, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.61953, std: 0.00578, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.61281, std: 0.00576, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.60946, std: 0.00458, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.67186, std: 0.00705, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.64909, std: 0.00587, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.63863, std: 0.00625, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 3, 'min_child_weight': 1},\n",
       " -0.590400929191564)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=610,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=3,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([243.46134448, 243.21021972, 239.86073217, 384.96516581,\n",
       "        382.80825911, 384.74193892, 541.15031075, 538.13923202,\n",
       "        529.88104305, 688.54538741, 671.16783004, 544.71640592]),\n",
       " 'std_fit_time': array([ 1.10328058,  1.99246361,  1.01795999,  1.46699386,  1.20846088,\n",
       "         1.44758924,  3.28981235,  7.60769357,  2.92981254,  5.23448299,\n",
       "         7.32916214, 38.55484342]),\n",
       " 'mean_score_time': array([1.57117171, 2.08516698, 1.8075038 , 3.66730285, 3.48376389,\n",
       "        4.78441515, 7.22678242, 6.74119582, 6.40927343, 9.32219882,\n",
       "        8.45284033, 6.78099475]),\n",
       " 'std_score_time': array([0.60501102, 0.76822341, 0.43270674, 0.96545303, 1.01078807,\n",
       "        0.68104449, 1.86958796, 1.67697405, 0.62936181, 1.56959259,\n",
       "        1.72860496, 1.75966995]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'split0_test_score': array([-0.58391306, -0.58279085, -0.58306411, -0.58268604, -0.58538501,\n",
       "        -0.58421381, -0.612464  , -0.60238007, -0.60168409, -0.65849929,\n",
       "        -0.64043787, -0.6289758 ]),\n",
       " 'split1_test_score': array([-0.58883351, -0.58873508, -0.58811014, -0.58994623, -0.58707465,\n",
       "        -0.58842848, -0.61747561, -0.61250838, -0.60791801, -0.67209236,\n",
       "        -0.64525507, -0.63697666]),\n",
       " 'split2_test_score': array([-0.59069691, -0.59317363, -0.59216088, -0.59228269, -0.59292971,\n",
       "        -0.59377353, -0.61749885, -0.61559114, -0.61012053, -0.67562337,\n",
       "        -0.64987303, -0.6402092 ]),\n",
       " 'split3_test_score': array([-0.59402748, -0.5942004 , -0.59343686, -0.59448276, -0.5926604 ,\n",
       "        -0.59158342, -0.62028506, -0.61380751, -0.61230637, -0.67409015,\n",
       "        -0.65235784, -0.63848878]),\n",
       " 'split4_test_score': array([-0.59453494, -0.59498615, -0.59568676, -0.59683215, -0.59772211,\n",
       "        -0.59479436, -0.62992648, -0.61974014, -0.61526537, -0.67897969,\n",
       "        -0.65755075, -0.64847779]),\n",
       " 'mean_test_score': array([-0.59040093, -0.59077697, -0.59049143, -0.59124563, -0.59115398,\n",
       "        -0.59055846, -0.61952937, -0.61280503, -0.60945852, -0.67185654,\n",
       "        -0.6490944 , -0.63862505]),\n",
       " 'std_test_score': array([0.00386984, 0.00454157, 0.00445618, 0.00485214, 0.00443796,\n",
       "        0.00385021, 0.00577769, 0.00575546, 0.0045843 , 0.00704824,\n",
       "        0.00587249, 0.00625159]),\n",
       " 'rank_test_score': array([ 1,  4,  2,  6,  5,  3,  9,  8,  7, 12, 11, 10], dtype=int32),\n",
       " 'split0_train_score': array([-0.53813077, -0.54059298, -0.54109573, -0.41832386, -0.43106688,\n",
       "        -0.44112673, -0.24649087, -0.28379399, -0.30889179, -0.10972786,\n",
       "        -0.15710104, -0.19279626]),\n",
       " 'split1_train_score': array([-0.53643072, -0.53871831, -0.53973319, -0.41696723, -0.4304346 ,\n",
       "        -0.43894554, -0.24338125, -0.27816761, -0.30436542, -0.10836864,\n",
       "        -0.15801853, -0.19393813]),\n",
       " 'split2_train_score': array([-0.53692615, -0.53915071, -0.54018925, -0.41631316, -0.42966247,\n",
       "        -0.43828242, -0.24266845, -0.27885836, -0.30439711, -0.10652806,\n",
       "        -0.15752156, -0.19353821]),\n",
       " 'split3_train_score': array([-0.53633961, -0.53913973, -0.53976617, -0.41729886, -0.43017378,\n",
       "        -0.4391001 , -0.24230881, -0.28072452, -0.30431221, -0.10945332,\n",
       "        -0.15811549, -0.19114743]),\n",
       " 'split4_train_score': array([-0.5363129 , -0.53828414, -0.53993863, -0.41382563, -0.4273726 ,\n",
       "        -0.43684674, -0.24435304, -0.27936514, -0.30492484, -0.10652266,\n",
       "        -0.15404091, -0.19117409]),\n",
       " 'mean_train_score': array([-0.53682803, -0.53917717, -0.54014459, -0.41654575, -0.42974207,\n",
       "        -0.43886031, -0.24384049, -0.28018193, -0.30537828, -0.10812011,\n",
       "        -0.15695951, -0.19251882]),\n",
       " 'std_train_score': array([0.00068826, 0.00077649, 0.0005023 , 0.00150729, 0.00126821,\n",
       "        0.00138491, 0.00149786, 0.00199087, 0.00177062, 0.00137917,\n",
       "        0.00150412, 0.00116787])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.590401 using {'max_depth': 3, 'min_child_weight': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VNX5+PHPM5OF7CxJQJYIQnADRIy4UBAUFBXDF/VLxdqqXyu2llprpYi2pVpr9ee31rbS1qXuVXEpGhTZFNFqkUVRAasE9CsBIRAhQAKBZJ7fH/dOmExmMgFyM1me94t5Zebec+995pLMM+ece88RVcUYY4xpiC/eARhjjGn5LFkYY4yJyZKFMcaYmCxZGGOMicmShTHGmJgsWRhjjInJkoUxxpiYLFkYY4yJyZKFMcaYmBLiHUBTyc7O1t69e8c7DGOMaVVWrly5XVVzYpVrM8mid+/erFixIt5hGGNMqyIi/9eYctYMZYwxJiZLFsYYY2KyZGGMMSamNtNnYYw5NAcOHKCkpIR9+/bFOxTTDDp06EDPnj1JTEw8rO0tWRjTTpWUlJCRkUHv3r0RkXiHYzykqpSVlVFSUkKfPn0Oax/WDGVMO7Vv3z66dOliiaIdEBG6dOlyRLVISxbGtGOWKNqPI/2/bvfJQlUp+/ujVBUXxzsUY4xpsdp9sjiwcSOl993HhnEX8cXFl/DNE09QvX17vMMyxpgWxdNkISJjReQzESkWkVuilJkoImtFZI2IPOMuGyUiq0Ie+0Tkv7yIMSkvj/wlb9F1uhPe1t/dzbqzRvLV5MmUv/oagb17vTisMe3ezp07+ctf/nJY295///1UVlY2cURNY+TIkYc9msTLL7/M2rVrD2lf+/btY+jQoZx00kmceOKJzJgx47COHYtnyUJE/MBM4HzgBGCSiJwQViYfmA4MU9UTgRsBVHWxqg5W1cHA2UAlsMCrWBOys+l85ZX0+edLHPPqHLpccw1V64rZfPPNrPvWcDZPv5WKpUvRQMCrEIxpd9pqsjgS4cmiMZKTk3nzzTf56KOPWLVqFfPmzWPp0qVNHpuXl84OBYpVdQOAiDwHjAdCz8S1wExV3QGgqqUR9nMp8LqqNstvRnK/fuTe9FNybvwJlctXUF70Crvnzad89mwSunUj66JxZBUWkpyf3xzhGNMsbp+zhrWbdzXpPk/onsmMi06Muv6WW25h/fr1DB48mDFjxpCbm8vzzz9PVVUVEyZM4Pbbb6eiooKJEydSUlJCTU0Nv/zlL9m6dSubN29m1KhRZGdns3jx4oj7T09P50c/+hGLFi2iU6dO3HXXXfz85z/nq6++4v7776ewsJAvv/yS7373u1RUVADwwAMPcOaZZzJ79mxmzpzJwoUL2bJlC2eddRZvv/023bp1q3ecvXv3cvXVV7N27VqOP/549oa0RixYsIAZM2ZQVVVF3759eeyxx0hPT6d37958+9vfro39mWeeobS0lKKiIpYsWcKdd97JSy+9BMALL7zA9ddfz86dO/n73//O8OHD6xxfREhPTwece2cOHDjgyYULXjZD9QA2hrwucZeF6g/0F5F3RWSpiIyNsJ/LgGc9ijEq8flIO20o3X/7W/Lf/Rc97vs9HY49lrJHH2PDRYVsmHAxZY89TvW2bc0dmjFtwt13303fvn1ZtWoVY8aMYd26dSxbtoxVq1axcuVK3n77bebNm0f37t356KOPWL16NWPHjuWGG26ge/fuLF68OGqiAKioqGDkyJGsXLmSjIwMfvGLX7Bw4UJmz57Nr371KwByc3NZuHAhH3zwAbNmzeKGG24AYMKECXTr1o2ZM2dy7bXXcvvtt0dMFAB//etfSU1N5eOPP+a2225j5cqVAGzfvp0777yTRYsW8cEHH1BQUMB9991Xu11mZibLli1jypQp3HjjjZx55pkUFhZy7733smrVKvr27QtAdXU1y5Yt4/777+f2228HYPPmzVxwwQW1+6qpqWHw4MHk5uYyZswYTjvttCP4n4nMy5pFpNSmEY6fD4wEegLviMgAVd0JICJHAQOB+REPIDIZmAyQl5fXNFFH4OvQgcwLLiDzgguoLitj12tzKS8qovSeeyi9917SzjyTrPGFZJxzDr7UVM/iMMYrDdUAmsOCBQtYsGABJ598MgB79uxh3bp1DB8+nJtvvplp06Yxbty4et+qG5KUlMTYsc73z4EDB5KcnExiYiIDBw7kyy+/BJxv4lOmTGHVqlX4/X4+//zz2u3//Oc/M2DAAE4//XQmTZoU9Thvv/12bZIZNGgQgwYNAmDp0qWsXbuWYcOGAbB//37OOOOM2u2C+5w0aRI//elPo+7/4osvBuCUU06pjbt79+7MnTu3tozf72fVqlXs3LmTCRMmsHr1agYMGNCo89RYXiaLEqBXyOuewOYIZZaq6gHgCxH5DCd5LHfXTwRmu+vrUdWHgIcACgoKwhORJxK6dKHz975L5+99l6oNGyh/pYjyOUVsnvpzfKmpZIwZQ9b4QlJPOw3x+5sjJGNaPVVl+vTpXHfddfXWrVy5krlz5zJ9+nTOPffc2lpBLImJibXNMT6fj+Tk5Nrn1dXVAPzhD3+ga9eufPTRRwQCATp06FC7/aZNm/D5fGzdupVAIIDPF70hJlKzj6oyZswYnn02csNI6DYNNRsF4/b7/bVxR9OxY0dGjhzJvHnzmjxZeNkMtRzIF5E+IpKE05xUFFbmZWAUgIhk4zRLbQhZP4k4NEE1VvIxx5D70xvpt2gRRz/1JJkXXsDuN9/kq/+5huJRZ7P13nvZ99nnsXdkTDuUkZHB7t27ATjvvPN49NFH2bNnD+B8UJeWlrJ582ZSU1O54ooruPnmm/nggw/qbXskysvLOeqoo/D5fDz11FPU1NQATtPP1VdfzTPPPMPxxx9fp/ko3IgRI/jHP/4BwOrVq/n4448BOP3003n33Xcpdu/hqqysrFNzmTVrVu3PYI3jcN7Xtm3b2LlzJ+D0nyxatIjjjjvukPbRGJ7VLFS1WkSm4DQh+YFHVXWNiNwBrFDVInfduSKyFqgBpqpqGYCI9MapmSzxKsamIj4fqaeeSuqpp9L1F79gz+LFlL9SxDdPPMk3f3+U5OOOI6uwkMwLLySxa268wzWmRejSpQvDhg1jwIABnH/++Vx++eW1H5rp6ek8/fTTFBcXM3XqVHw+H4mJifz1r38FYPLkyZx//vkcddRRDfZbxHL99ddzySWX8MILLzBq1CjS0tIAuOuuuxg+fDjDhw9n8ODBnHrqqVx44YUcf/zx9fbxwx/+kKuvvppBgwYxePBghg4dCkBOTg6PP/44kyZNoqqqCoA777yT/v37A1BVVcVpp51GIBCorX1cdtllXHvttfzpT3/ixRdfjBr35s2b+f73v8/cuXP5+uuvufLKK6mpqSEQCDBx4kTGjRt32OckGlFtltYbzxUUFGhLmymv+ptv2DX3dcqLitj38cfg85F2+ulO/8bo0fjcX0xj4uHTTz+N+OFnvBec2TM7O7tZjxvp/1xEVqpqQaxtbdRZDyV07kznK75D5yu+Q9WGLyifU8SuojlsnnYLkppKxuhzyCocT9oZp1v/hjGmRbNk0UySj+lD7k9+Qs6Pf8zeDz6g/JUids2bx66iOSTk5JA5bhxZ4wvp4EFbozFt2WmnnVbbzBP01FNPMXDgwCY9zvz585k2bVqdZX369GH27NmHvK/gVU2tiTVDxVGgqoo9by2hvKiIPW+/DQcOkNy/P1njC8kcN47Erl3jHaJpw6wZqv2xZqhWypecTOZ555J53rlU79jBrtdfZ9crRZTe+7+U/u/vST39NLIKx5MxZgz+dOvfMMbET7sfdbalSOjUic6XX07vWc/Rd97rZP/whxwo2cTX06ezbvhwNt08lT3vvIPGuM7aGGO8YDWLFiipd29ybvgx2T+ewt4PPzzYv/Hqq/hzssm64EKyxheSfPzxNnmNMaZZWLJowUSE1CFDSB0yhK633cqet96ivKiIb555hm+eeILk/H5kFhaSddFFJEYZt8YYY5qCNUO1Er6kJDLPPZdeDzxA/ttL6DbjV/jS0tn2+/soHnU2/3fV1ez852xq3DtgjWnp2uoQ5c09nwU4920MHDiQwYMHU1AQs6/6sFiyaIUSOnWi06RJ9H7uWfoumE/2j37Egc2b+frWW1n3reFsuuln7FmyxPo3TIvWVpPFkTic+SyCFi9ezKpVqw47UcVizVCtXFJeHjlTfkT2j65n76pVlBcVsXvu6+yaOxd/ly5kXngBWYXj6XDiCda/YaJ7/RbY8knT7rPbQDj/7qirbT6LppnPorlYzaKNEBFSTz6Zo2bMIP+dt+n5wJ9JHTKEnc8+x5eXXsqGcRex/cGHOLA5fOBfY+LD5rNouvksRIRzzz2XU045hYceeugI/leis5pFGyRJSWSMHk3G6NHUlJez6/V5lBcVse0Pf2DbH/5A6tChzvhU552H351hy7RzDdQAmoPNZ3Fk81m8++67dO/endLSUsaMGcNxxx3HiBEjGnWeGsuSRRvnz8qi02XfptNl32b/xo2Uz5nDrleK+Pq2X7Dljt+Qcc7ZZBYWkj5sGJKYGO9wTTtl81kc2XwW3bt3B5ya0oQJE1i2bFmTJwtrhmpHknr1Iuf66zlm3uv0nvUcHS+5hIr3/k3JD37IurNGsuXO37L3k09oK0PAmJbN5rNomvksKioqarepqKhgwYIFTT7xEVjNol0SEVJOOomUk06i6y3T2POvf1H+ShE7n3+eHU8/TVKfPmSNd+/f6BE+bboxTcPms2ia+Sy2bt3KhAkTACfJXX755bXNb03J04EERWQs8EecyY8eUdV6DaMiMhH4Nc783B+p6uXu8jzgEZwJkBS4QFW/jHas1jiQYEtTs2sXu+Y5/Rt7VziddKkFBWSOLyTzvPPwZ2bGOULTlGwgwfix+SzqBuAHZgJjcObaXi4iRaq6NqRMPjAdGKaqO0QkdBq5J4HfqupCEUkHAl7Fahz+zEw6TZxIp4kT2V9Swq45cyh/pYgtv/wVW39zJ+lnn01WYSHpw79l/RvGtDNeNkMNBYpVdQOAiDwHjAdC7zi5FpipqjsAVLXULXsCkKCqC93ldltyM0vq2ZPsH/6QLj/4AftWr3bGp3rtNXbPm4e/UycyL7iArMKL6DBokN2/YeLK5rNoHl4mix7AxpDXJcBpYWX6A4jIuzhNVb9W1Xnu8p0i8k+gD7AIuEVVazyM10QgIqQMHEjKwIF0nfZzp3+jqIidL7zAjn/8g6TevcksvIiswkKSevaMd7imHXr//feb5TjnnXce5513XrMcqyXyMllE+roZ3kGSAOQDI4GewDsiMsBdPhw4GfgKmAVcBfy9zgFEJgOTAfLy8pouchORJCaSMWoUGaNGUbN7N7vnz6f8lSK2/+nPbP/Tn0k55RSyCgvJHHse/qyseIdrjGlCXl46W4LTOR3UEwi/fbgEeEVVD6jqF8BnOMmjBPhQVTeoajXwMjAk/ACq+pCqFqhqQU5OjidvwkTmz8ig46WXcvRTT9LvjUXk3HgjNTt2sGXGDNZ9azglN/yE3W+8ge7fH+9QjTFNwMuaxXIgX0T6AJuAy4DLw8q8DEwCHheRbJzmpw3ATqCTiOSo6jbgbMAudWqhEnv0IPsH19HlusnsW72G8iK3f2PBAvwdO5J5wflkXnQRKYMHW/+GMa2UZzULt0YwBZgPfAo8r6prROQOESl0i80HykRkLbAYmKqqZW7fxM3AGyLyCU6T1sNexWqahtO/MYBut91K/pK36PXg30g780x2vvRP/m/S5awfO5ZtD8xk/1dfxTtU0wK01VFnm3uI8s8++4zBgwfXPjIzM7n//vsP6/gN8fSmPFWdC8wNW/arkOcK3OQ+wrddCAzyMj7jHUlMJP2ss0g/6yxq9uxh9/wFlBcVsX3mTLY/8AApJ59M1vhCMseOxd+xY7zDNXEQTBbXX3/9IW97//33c8UVV5CamupBZPHz8ssvM27cOE444YRGb3PssceyatUqAGpqaujRo0ftTXpNyYb7MJ7zp6fT8ZKLOfqJx+n35hvk3HQTNbt3seXXt7Nu+AhKfvxjdi1cSMD6N9qV0CHKp06dyr333supp57KoEGDmDFjBuAMX3HhhRdy0kknMWDAAGbNmsWf/vSn2iHKR40aFXX/6enpTJs2jVNOOYXRo0ezbNkyRo4cyTHHHENRURHgXMI6fPhwhgwZwpAhQ3jvvfcAmD17NqNHj0ZV+frrr+nfvz9btmyJeJy9e/dy2WWXMWjQIL797W/XG6L8jDPOYMiQIfz3f/937XAmvXv3Ztq0aQwdOpShQ4dSXFzMe++9R1FREVOnTmXw4MGsX78ecIYoHzp0KP379+edd95p8Jy+8cYb9O3bl6OPPrqR/wuNZ8N9mGaVeNRRZE++li7Xfp99a9eyq6iI8tfmsnvhInxZWWSeP5aswvGknGz9G83pnmX38J9v/tOk+zyu83FMGzot6vq7776b1atXs2rVKhYsWMCLL77IsmXLUFUKCwt5++232bZtG927d+e1114DnLGcsrKyuO+++1i8eHGDd0AHhyi/5557mDBhQu0Q5WvXruXKK6+ksLCwdojyDh06sG7dOiZNmsSKFSuYMGECL730EjNnzmTevHmNHqL8448/ZsgQ51qc0CHK09LSuOeee7jvvvtqB0IMDlH+5JNPcuONN/Lqq69SWFjIuHHjuPTSS2v3HxyifO7cudx+++0sWrSoznAfoZ577rkGR8g9EpYsTFyICCknnkjKiSeSO3UqFe+9R/krRZS//Ao7n5tFYl4eWRddRFbhRSR58C3JtCw2RPmRDVEe3H9RURG/+93vYp6bw2HJwsSdJCSQPmIE6SNGULOngt0LF1Je9Arb//IXts+cScpJJznjU51/PgmdOsU73DapoRpAc7Ahyo9siHKA119/nSFDhtC1a9eoZY6E9VmYFsWfnkbHCf/F0Y89Rr/Fb5J7888IVFay9Y7fsG7EWWz80RR2zV9g/RttgA1R3jRDlAc9++yznjVBgdUsTAuW2K0bXb7/fTpfcw1V//mP00z12qvseeMNfJmZZI4dS9b4QlKGDLH+jVbIhihvmiHKwUlECxcu5MEHHzzscxGLp0OUNycborx90OpqKv69lPKiInYvWoTu3Utiz55kBcen6t073iG2GjZEefzYEOXGeEwSEkgf/i3Sh3+LQEUFuxctcsan+tuDbP/LX+lw0iBnfKoLLrD+DWOakCUL02r50tLIGj+erPHjObC1lF2vvkp5URFbf3MnW393N+nDh5M1vpD0UaPwuZ2Epu2xIcqbhzVDmTZn32efOfNvzJlD9bZt+DIyyBx7HlmFhaSccgrSwFUt7Yk1Q7U/1gxlTIgOxx5Lh59PJfdnN1GxdGntjX87X3iRxO7d3fk3xpN8TJ94h2pMq2HJwrRZ4veTPmwY6cOG0W1GpdO/UTSHsocepuxvD9Jh4EC3f+N8Erp0iXe4xrRoVh837YIvNZWswkLyHnmYfm8tJnfaNLS6mq2//a1z/8Z1P2DX3LkE9u2Ld6jGtEhWszDtTmJuLl2uvoouV1/Fvs8/d5qp5rzKniVL8KWnk3HeuWQVjif11ALr3zDGZX8Jpl3r0L8/uTffTL833yDvsUfJGD2a3a/P46srr6R49GhK7/sDVe7on6Zp2XwW9R3OfBYAf/zjHxkwYAAnnniiJ3NZgCULYwCnfyPtjDPofvfvyP/XO3S/916S+/aj7JFH2HDhOL645FK+efJJqsvK4h1qm9FWk8WRCE8WjbF69Woefvhhli1bxkcffcSrr77KunXrmjw2T5OFiIwVkc9EpFhEbolSZqKIrBWRNSLyTMjyGhFZ5T6KvIzTmFC+1FSyLhpH3sMPkb/kLXJvmYZqgK13/Y51I87iq+uuo/y11wiEzFtgDp3NZ9E081l8+umnnH766aSmppKQkMBZZ511WPd+xOJZn4WI+IGZwBigBFguIkWqujakTD4wHRimqjtEJDdkF3tVdbBX8RnTGAk5OXS56iq6XHUVVevWUV40h/I5c9j8s5vxpaWRce65ZI0vJHXo0Fbdv7Hlrruo+rRp57NIPv44ut16a9T1Np9F08xnMWDAAG677TbKyspISUlh7ty5FBTEvG3ikHnZwT0UKFbVDQAi8hwwHgitY10LzFTVHQCqWuphPMYckeT8fHJ/dhM5P72RymXLnfGp5s+nfPZsErp1I+uicWQVFpKcnx/vUFsdm8/i8OezOP7445k2bRpjxowhPT2dk046iYSEpv9o9zJZ9AA2hrwuAU4LK9MfQETeBfzAr1V1nruug4isAKqBu1X15fADiMhkYDJAXl5e00ZvTBTi85F2+mmknX4agV/+gt1vvkl5URFljz5G2cOPkHzC8WQVFpJ14YUk5OTEO9xGaagG0BxsPosjm8/immuu4ZprrgHg1ltvpWfPnlH3d7i8rDdHevfhY4skAPnASGAS8IiIdHTX5bm3oF8O3C8ifevtTPUhVS1Q1YKcVvJHadoWX0oKWRdeSN6DD5K/5C263jodER+ld9/DurNG8tW1kymf8yrVO3bEO9QWx+azaLr5LEpLnUaZr776in/+85+ezGvhZc2iBOgV8ronsDlCmaWqegD4QkQ+w0key1V1M4CqbhCRt4CTAbuG0bRYCdnZdP7e9+j8ve9RtX69M//Gq3PYPHUqAP4uXUju25fkfv1Izu9Hcr9+JPXr125Hx7X5LJpuPotLLrmEsrIyEhMTmTlzJp08+J3ybCBBEUkAPgfOATYBy4HLVXVNSJmxwCRVvVJEsoEPgcFAAKhU1Sp3+b+B8aGd4+FsIEHTEmkgwN4PPmDvJ6upKl7H/uL1VBUXE6ioqC3j79LFSSB9+zZrErGBBOPH5rMIoarVIjIFmI/TH/Goqq4RkTuAFapa5K47V0TWAjXAVFUtE5EzgQdFJIDTVHZ3Q4nCmJZKfD5SCwpIDbk6RVWp3rKFquJiqorXU1W8jqriYspfeSVyEgnWRPr2bdc1ERNfNkS5MS1EnSSyrtj5ub6Y/cXr6yaR7Ox6NZHkfv3wd+zYwN7rays1i9Y4n0W8HEnNwpKFMS2cqlL99ddUrV9/MIkUF7O/uJhAyF3MtUmk9tG3wSTSVpKFabwW2QxljGkaIkJi9+4kdu9Oesh9BrVJJFgTWe/0h5T/8591k0hONsl9+9Vr0gruo6HLNk3bcaQVA0sWxrRSdZLIiBG1y1WV6s2b69ZE1q+vl0Sqf3EbW1Tp0rEjvg4dkORkfMnJiAc3dJn4UlXKysrq3EdyqOy3wpg2RkRI7NGDxB496iaRQOBgTaR4PZXF6/mmQwe2hXeY+3xIQgKSmOgkDvdnax7OxECHDh2O6GY9SxbGtBPi8x1MImedRXBuwLpJJKRJa/16NKQmkpCTQ3K+c1lvct+D94r4MzPj84ZMs7JkYUw7F55EgjQQ4MDmr537Q0KatHa+8CIaMrJqQm4uyf2cy3qdfpF8kvv1tSTSxliyMMZEJD4fST17kNSzB4wcWbu8ThKpvVckWhLpR5J7VZYlkdbNkoUx5pA0nEQ2117WG2zOipZEwpu0/BkZcXg3prEsWRhjmoSTRHqS1LNn5CSyrm5z1o7nX6ibRLp2rb0/JCnkfhFLIi2DJQtjjKfqJJGQme1Ck0htbaR4PTtmPY/u21db7mAScWsj7mCMlkSalyULY0xchCaRjPAksmmT24x1sElrx6xZdZNIt24RR/H1p6fH4+20eZYsjDEtivh8JPXqRVKvXmScHSWJFBezf30DSSRsyBNLIkfOkoUxplWImkRqapwk4l6VFRzFt/LZ5WjIAIMJRx1VvybSt68lkUayZGGMadXE7ycpL4+kvLwoSaS4tkkrahIJr4n07Yc/PS0eb6fFsmRhjGmT6iaRs2uX10si7lDwlcuW1U0i3Y+qNwBj0jF9220S8TRZuDPh/RFn8qNHVPXuCGUmAr/GmZ/7I1W9PGRdJvApMFtVp3gZqzGmfWgwiZSUhExK5dZEIiWR4P0hIaP4+tLadhLxLFmIiB+YCYzBmWt7uYgUhc54JyL5wHRgmKruEJHcsN38BljiVYzGGBMkfj9JRx9N0tFHk3HOObXL6ySRkKHgK5e+j+7fX1uuNon0yz/YpNWGkoiXNYuhQLGqbgAQkeeA8UDo9KjXAjNVdQeAqpYGV4jIKUBXYB4Qc2IOY4zxQoNJZOPGepNShSeRxO7dScoPq4kcc0yrSyJeJosewMaQ1yXAaWFl+gOIyLs4TVW/VtV5IuIDfg98FzgHY4xpYcTvJ6l3b5J6946cRIKj+LpNWpX/Xho5iYSMm9WSk4iXySLS9FvhUzUlAPnASKAn8I6IDACuAOaq6saGZvESkcnAZIC8vLwmCNkYY45MnSQyenTtcq2uDukTOdikVS+J9OgRNvhiP5L7HoMvNTUeb6dWzGQhIn2BElWtEpGRwCDgSVXdGWPTEqBXyOuewOYIZZaq6gHgCxH5DCd5nAEMF5HrgXQgSUT2qOotoRur6kPAQ+DMwR3rvRhjTLxIQkLUJLLfrYmEjp1V+d6/0QMHassl9ugRMuRJv2ZPIhJrXlYRWYXTZ9AbmA8UAceq6gUxtksAPsdpRtoELAcuV9U1IWXGApNU9UoRyQY+BAarallImauAglhXQxUUFOiKFSsafC/GGNNa1EkiIc1Z+zdsOJhE3FkR0844g6N+c8dhHUdEVqpqzH7hxjRDBVS1WkQmAPer6p9F5MNYG7nbTMFJMH7gUVVdIyJ3ACtUtchdd66IrAVqgKmhicIYY9orSUgguU8fkvv0gTFjapdrdTX7v9pYZ9wsSUryPp5G1CzeB+4HbgMuUtUvRGS1qg7wPLpDYDULY4w5dI2tWTRmBvarcfoQfusmij7A00caoDHGmNYjZjOUexPdDQAi0gnIiHQntjHGmLYrZs1CRN4SkUwR6Qx8BDwmIvd5H5oxxpiWojHNUFmqugu4GHhMVU8BRsfYxhhjTBvSmGSRICJHAROBVz2OxxhjTAvUmGRxB84lrutVdbmIHAOs8zYsY4wxLUljOri7cyV7AAAZ7UlEQVRfAF4Ieb0BuMTLoIwxxrQsjeng7ikis0WkVES2ishLItKzOYIzxhjTMjSmGeoxnCE+uuOMJDvHXWaMMaadaEyyyFHVx1S12n08DuR4HJcxxpgWpDHJYruIXCEifvdxBWDjNxljTDvSmGTxPziXzW4BvgYuxRkCxBhjTDsRM1mo6leqWqiqOaqaq6r/hXODnjHGmHaiMTWLSG5q0iiMMca0aIebLKLPdWqMMabNOdxk0agpTEVkrIh8JiLFInJLlDITRWStiKwRkWfcZUeLyEoRWeUu/8FhxmmMMaYJRL2DW0R2EzkpCJASa8ci4gdmAmNw5tpeLiJF7pDnwTL5wHRgmKruEJFcd9XXwJnuvN/pwGp32/A5vI0xxjSDqMlCVTOOcN9DgWJ3eBBE5DlgPLA2pMy1wExV3eEes9T9uT+kTDKHXwMyxhjTBLz8EO4BbAx5XeIuC9Uf6C8i74rIUhEZG1whIr1E5GN3H/dYrcIYY+LHy2QRqRM8vFkrAcgHRgKTgEdEpCOAqm5U1UFAP+BKEela7wAik0VkhYis2LZtW5MGb4wx5iAvk0UJ0CvkdU8gvHZQAryiqgdU9QvgM5zkUcutUawBhocfQFUfUtUCVS3IybERSIwxxiteJovlQL6I9BGRJOAynAEJQ70MjAIQkWycZqkN7ki3Ke7yTsAwnERijDEmDhozRPluEdkV9tjoDlt+TLTtVLUamIIzcdKnwPOqukZE7hCRQrfYfKBMRNYCi4GpqloGHA+8LyIfAUuA/1XVT47srRpjjDlcotrwLRMicjtO89EzOP0QlwHdcL7p/1BVR3ocY6MUFBToihUr4h2GMca0KiKyUlULYpVrTDPUWFV9UFV3q+ouVX0IuEBVZwGdjjhSY4wxLV5jkkXAvcva5z4mhqxr1J3cxhhjWrfGJIvvAN8FSt3Hd4Er3A7oKR7GZowxpoWIegd3kHsH9kVRVv+racMxxhjTEjXmaqie7pVPpSKyVUReEpGezRGcMcaYlqExzVCP4dwf0R1nuI457jJjjDHtRGOSRY6qPqaq1e7jccBulzbGmHakMcliu4hcISJ+93EFUOZ1YMYYY1qOxiSL/wEmAltw5pm4FLjay6CMMca0LDGThap+paqFqpqjqrmq+l/Axc0QmzHGmBbicAcSvKlJozDGGNOiHW6yiDRXhTHGmDbqcJOFDfNhjDHtSNQ7uEVkN5GTggApnkVkjDGmxYmaLFQ1ozkDMcYY03J5OVMeIjJWRD4TkWIRuSVKmYkislZE1ojIM+6ywSLyb3fZxyLybS/jNMYY07CYAwkeLhHxAzOBMThzbS8XkSJVXRtSJh+YDgxT1R0ikuuuqgS+p6rrRKQ7sFJE5qvqTq/iNcYYE52XNYuhQLGqblDV/cBzwPiwMtcCM1V1B4Cqlro/P1fVde7zzThDo9sQI8YYEydeJosewMaQ1yXuslD9gf4i8q6ILBWRseE7EZGhQBKw3rNIjTHGNMizZigi34sRfnVVApAPjAR6Au+IyIBgc5OIHAU8BVypqoF6BxCZDEwGyMvLO6wgqwPVPL32aXJTc+ma1tX5mdqVJH/SYe3PGGPaIi+TRQnQK+R1T2BzhDJLVfUA8IWIfIaTPJaLSCbwGvALVV0a6QDufOAPARQUFBzWvR9le8v4/crf11veKblTneQR/Fn7PK0r6YnpiNj9icaYts/LZLEcyBeRPsAm4DLg8rAyLwOTgMdFJBunWWqDiCQBs4EnVfUFD2MkNzWX9ya9R2llKVsrtrK10nmUVpbW/ly9fTXf7Pum3rYpCSm1CSRiYknrSucOnfGJpxedGWOM5zxLFqpaLSJTgPmAH3hUVdeIyB3AClUtctedKyJrgRpgqqqWucOgjwC6iMhV7i6vUtVVTR2niJCRlEFGUgZ9O/aNWm5/zf46CSQ8sSzbsoztldup1uo62yVIAjmpOXUSSbe0bnVe56bmWrOXMaZFE9W2MXJHQUGBrlixIq4xBDTAN/u+qV9DqThYU9lauZW91Xvrbdu5Q+d6NZNgc1ew9pKelB6Hd2WMactEZKWqFsQq52UzVLvjEx/ZKdlkp2RzIidGLKOq7D6wm9KK0joJJJhYtlRs4eNtH7Ojake9bVMTUuslkNrEkub8tGYvY4wXLFk0MxEhMymTzKRM+nXqF7VcVU1VnVpJeGJZtmUZ2yq3UaM1dbZL8CWQm5Jb27xVL7GkdSU3JZdEf6LXb9UY04ZYsmihkv3J9MroRa+MXlHL1ARqnGavYBIJSyyf7/icdza9E7XZKzSJhF463C3V6VOxZi9jTJAli1bM7/OTk5pDTmoOAxgQsUyw2Su83yT4+uuKr1m1bRU7q+qPpJKWmNbgpcO5qbnW7GVMO2HJoo0LbfbK75Qftdy+6n1sq9wW8dLhrRVbef/r99m+d3vUZq96lw6nhdRarNnLmFbPkoUBoENCB3pl9qJXZsPNXmX7yqLek/Kfb/7Dko1L2Fezr962oc1e0RJLWmKal2/RGHMELFmYRvP7/LX9GwOyozd77dq/K+o9KZsrNvPhtg8pryqvt21aYlq9S4fD70np1KGTNXsZEweWLEyTEhGykrPISs6K2exVJ6GE3ZPy76//zfa92wmEDQmW6Es82CEfdulwsGM+JyXHmr2MaWKWLExcdEjoQF5mHnmZ0QeArA5UU7a3rPYKry2VW+oklrVla3lr41v1mr0EcZq90qJ3zHdL7UZqYqrXb9OYNsOShWmxEnwJTn9GWteoZYLNXqFXeIXWVjbt2cSHpZGbvdIT0+slkfDE0im5kw0WaQyWLEwrF9rs1b9T/6jl9lbvrXO1V3hiWb95fYPNXpFqJ8Fl2anZJPqs2cu0bZYsTLuQkpDS6Gav0JpJaGJZXbaa0o2lVNVU1dlOELqkdGmwY75jckfSE9OtL8W0WpYsjHEdSrPXloot9Zq8tlZuZePujazcupJd+3dF3D7Jl0RaYhppiWmkJ6WTmpBKelL6wWWJkZ/XeZ2URlpCGn6f36tTYUw9liyMOQShzV7Hdj42arm91XsPdsxXbGHX/l1UHKhgz4E9VB6oZM+BPVTsr6CiuoJtldv48sCXVByooOJARcT7VCJJSUipTSKpianRk4ubmNISnEQTWj49MZ2UhBTrlzExWbIwxgMpCSkcnXk0R2cefcjbVgeqaxNHaHKpfb5/T531wecVByrYtGdTnXXVgeqYxxMkapKpszws4YSWDyafZH+yJZ42ypKFMS1Mgi+htvZypPbX7K+TUPbs30NltZNw6tRyIiSfbXu31Xkd3vkfMXZJOFjLcZvLIiWXSIkmfL3177QsniYLERkL/BFnprxHVPXuCGUmAr8GFPhIVS93l88DTgf+parjvIzTmLYqyZ9EZ39nOnfofET7UVX2Vu+tTTSRkkt4LWfP/j1UVFdQvq+cTbs31SamyurKxsXuS6rTnxOeXKImGjdJBbdNTUi1/p0m4FmyEBE/MBMYA5QAy0WkSFXXhpTJB6YDw1R1h4jkhuziXiAVuM6rGI0xjSMipCamkpqYSnZK9hHtK6CBg4kjvIktPPm4/ToV+53XpZWldZrnDqV/J2aNJkJiCn/envt3vKxZDAWKVXUDgIg8B4wH1oaUuRaYqao7AFS1NLhCVd8QkZEexmeMiQOf+EhPSm+S+VIOBA5QeaAyci1nf0XUGlDlgcrD6t/xiY+0hLSIiaahK9siXXCQ5EtqVYnHy2TRA9gY8roEOC2sTH8AEXkXp6nq16o6r7EHEJHJwGSAvLzo188bY9qmRF9i0/fvuLWZhi4kCE8+tf077raN7d8JvzqtsYkmmJiC2zbHTaFeJotIKVMjHD8fGAn0BN4RkQGqWn8mnkg7U30IeAigoKAgfN/GGNNoTd2/01ByiXTBQcWBCnbu20nJ7pJD7t85Kecknr7g6SOKOxYvk0UJEDo5Qk9gc4QyS1X1APCFiHyGkzyWexiXMcZ4JrR/J4ecI9pXTaCGvdV7Y15I0KlDpyaKPjovk8VyIF9E+gCbgMuAy8PKvAxMAh4XkWycZqkNHsZkjDGtht/nb7L+nSPl2SwyqloNTAHmA58Cz6vqGhG5Q0QK3WLzgTIRWQssBqaqahmAiLwDvACcIyIlInKeV7EaY4xpmKi2jab+goICXbFiRbzDMMaYVkVEVqpqQaxyNj+lMcaYmCxZGGOMicmShTHGmJgsWRhjjInJkoUxxpiYLFkYY4yJyZKFMcaYmCxZGGOMicmShTHGmJgsWRhjjInJkoUxxpiYLFkYY4yJyZKFMcaYmCxZGGOMicmShTHGmJg8TRYiMlZEPhORYhG5JUqZiSKyVkTWiMgzIcuvFJF17uNKL+M0xhjTMM+mVRURPzATGIMz1/ZyESlS1bUhZfKB6cAwVd0hIrnu8s7ADKAAUGClu+0Or+I1xhgTnZc1i6FAsapuUNX9wHPA+LAy1wIzg0lAVUvd5ecBC1X1G3fdQmCsh7EaY4xpgJfJogewMeR1ibssVH+gv4i8KyJLRWTsIWxrjDGmmXjWDAVIhGXhE34nAPnASKAn8I6IDGjktojIZGAyQF5e3pHEaowxpgFe1ixKgF4hr3sCmyOUeUVVD6jqF8BnOMmjMduiqg+paoGqFuTk5DRp8MYYYw7yMlksB/JFpI+IJAGXAUVhZV4GRgGISDZOs9QGYD5wroh0EpFOwLnuMmOMMXHgWTOUqlaLyBScD3k/8KiqrhGRO4AVqlrEwaSwFqgBpqpqGYCI/AYn4QDcoarfeBWrMcaYholqva6AVqmgoEBXrFgR7zCMMaZVEZGVqloQq5zdwW2MMSYmSxbGGGNismRhjDEmJksWxhhjYrJkYYwxJiZLFsYYY2KyZGGMMSYmSxbGGGNismRhjDEmJksWxhhjYrJkYYwxJiZLFsYYY2KyZGGMMSYmSxbGGGNismRhjDEmJi/n4EZExgJ/xJn86BFVvTts/VXAvcAmd9EDqvqIu+4e4EJ3+W9UdZYnQe7dAU9dHBpUaITRlx122WjbN7DPRm1/pMc/lPfaWmKViKvjE6vPefjcn+IPWRbyvN5rf+Ttwreps8wf/Xj19i1RtnPXHcp2dcpE+H0xrZpnyUJE/MBMYAzOnNrLRaRIVdeGFZ2lqlPCtr0QGAIMBpKBJSLyuqruavpAfZDaxX0RMhFU7aRQkZYdatko2wdfN/ZY7jINf04guDBks7rPnX91939wdSDCstD9a0ho7hKtH2toXJHei9bup4FzVxtDaJm67+PgAq2/vt5+67wLJCRWCTuv0fcReb1o2H6ibO+sV3waQFB81CCqCIqfGtoiRVDxoeIDfBGfq5t01E02zjbuczfpBMsQoTy1rw8mLJXw5OhD8SHiQ+slt9BkWzfhic8H+MFXN2FKeFJ0l4kvdHtBxF/73OdLcPfpR92fIuJsE2F7nxuX+BKcdf4Ixw1/DwkpkNHV0/9TL2sWQ4FiVd0AICLPAeOB8GQRyQnAElWtBqpF5CNgLPB8UwdZHkjlB5U3oygB52+agPuBFVDnA1dDXgfcz2V114WWjbztwefBdfW2DQSXO+sC7gd7QOuXbSMTG7YoIs73f5+I81zE+YzA/RlcDvh8Uq9s8HWdsuKWwVkGB3+P1P3/Df5+aCCAjwBoAB81wV8EN6kEEAKIHiwjBPCpustrENT5qYqPgJOIpAafKmhN7TK/BJOV4sfZj58Azsdp8GfYOqm7znlefztnG+d18Dj19x2oLRdpnU8O7qveujrHqT5YRsLXhW8fIU4Jlon8HoLr6uxLWvYf3rrE48i/7X1Pj+FlsugBbAx5XQKcFqHcJSIyAvgc+KmqbgQ+AmaIyH1AKjCKxiWZw1IdCBz8o/cJCSIRPgwif0AEPxQI+XAI/dDwSYQPFyHKh1HwdbB8/bLU+yCre6zgurofegc/5BqKK/TYhL2/iNv6Ip+LemXd94Abc53zFr5tWNlgXLVl631Y1z2Pte/fdzABNFi2nTWXRPryEUxcEPIlh7Dk1tDysP0EvwARYXnwyxB1lrlfjgJ1Y6ldHnac4LbB5TVAdbDmHFxOpHhDEnSk5WHxBjTkPWgA1QAEatBAAAgggQBoDQQCKM46AgEg+NMpo4Eap/auTnnRgLtTd3mgBghAwEnSBPerzpcA1PmyIFqDhi6r3aaGlKxc8r38xcHbZBHprzA8Pc8BnlXVKhH5AfAEcLaqLhCRU4H3gG3Av4HqegcQmQxMBsjLyzusILNSE3nhB2ce1rbGtDa1STzin6cx0Xl5NVQJ0CvkdU9gc2gBVS1T1Sr35cPAKSHrfquqg1V1DE7iWRd+AFV9SFULVLUgJyenyd+AMcYYh5fJYjmQLyJ9RCQJuAwoCi0gIkeFvCwEPnWX+0Wki/t8EDAIWOBhrMYYYxrgWTOUqlaLyBRgPs6ls4+q6hoRuQNYoapFwA0iUojTxPQNcJW7eSLwjtuevAu4wu3sNsYYEwdS5xLIVqygoEBXrFgR7zCMMaZVEZGVqloQq5zdwW2MMSYmSxbGGGNismRhjDEmJksWxhhjYmozHdwisg34vyPYRTawvYnCaUoW16GxuA6NxXVo2mJcR6tqzBvV2kyyOFIisqIxVwQ0N4vr0Fhch8biOjTtOS5rhjLGGBOTJQtjjDExWbI46KF4BxCFxXVoLK5DY3EdmnYbl/VZGGOMiclqFsYYY2JqV8lCRB4VkVIRWR1lvYjIn0SkWEQ+FpEhLSSukSJSLiKr3MevmimuXiKyWEQ+FZE1IvKTCGWa/Zw1Mq5mP2ci0kFElonIR25ct0cokywis9zz9b6I9G4hcV0lIttCztf3vY4r5Nh+EflQRF6NsK7Zz1cjYornufpSRD5xj1tvMDxP/x6dWaLaxwMYgTO39+oo6y8AXseZP+N04P0WEtdI4NU4nK+jgCHu8wyc2QxPiPc5a2RczX7O3HOQ7j5PBN4HTg8rcz3wN/f5ZThz0LeEuK4CHmju3zH32DcBz0T6/4rH+WpETPE8V18C2Q2s9+zvsV3VLFT1bZyh0KMZDzypjqVAx7A5N+IVV1yo6teq+oH7fDfOfCM9woo1+zlrZFzNzj0He9yXie4jvFNwPM6MkAAvAueIx3O7NjKuuBCRnsCFwCNRijT7+WpETC2ZZ3+P7SpZNEKkecPj/iHkOsNtRnhdRE5s7oO71f+Tcb6VhorrOWsgLojDOXObL1YBpcBCVY16vtSZo6Uc6NIC4gK4xG26eFFEekVY74X7gZ8DgSjr43G+YsUE8TlX4CT5BSKyUpxppcN59vdoyaKuxswbHg8f4NySfxLwZ+Dl5jy4iKQDLwE3ququ8NURNmmWcxYjrricM1WtUdXBONMIDxWRAWFF4nK+GhHXHKC3qg4CFnHw27xnRGQcUKqqKxsqFmGZZ+erkTE1+7kKMUxVhwDnAz8SkRFh6z07X5Ys6oo5b3g8qOquYDOCqs4FEkUkuzmOLSKJOB/I/1DVf0YoEpdzFiuueJ4z95g7gbeAsWGras+XiCQAWTRjE2S0uFS1TFWr3JcPA6c0QzjDgEIR+RJ4DjhbRJ4OK9Pc5ytmTHE6V8Fjb3Z/lgKzgaFhRTz7e7RkUVcR8D33ioLTgXJV/TreQYlIt2A7rYgMxfl/K2uG4wrwd+BTVb0vSrFmP2eNiSse50xEckSko/s8BRgN/CesWBFwpfv8UuBNdXsm4xlXWLt2IU4/kKdUdbqq9lTV3jid12+q6hVhxZr1fDUmpnicK/e4aSKSEXwOnAuEX0Hp2d+jZ3Nwt0Qi8izOVTLZIlICzMDp7ENV/wbMxbmaoBioBK5uIXFdCvxQRKqBvcBlXn/AuIYB3wU+cdu7AW4F8kJii8c5a0xc8ThnRwFPiIgfJzk9r6qvSt155/8OPCUixTjfkC/zOKbGxnWDiBQC1W5cVzVDXBG1gPMVK6Z4nauuwGz3O1AC8IyqzhORH4D3f492B7cxxpiYrBnKGGNMTJYsjDHGxGTJwhhjTEyWLIwxxsRkycIYY0xMliyMMcbEZMnCmGbmDjN9WHeTu8Njd2+KfRlzKCxZGNO6XAV0j1XImKZmycK0WyLSW0T+IyKPiMhqEfmHiIwWkXdFZJ2IDHUf74kzEc57InKsu+1NIvKo+3ygu31qlON0EZEF7j4eJGSwNxG5QpyJiVaJyIPuXdaIyB4R+b2IfCAib7hDdlwKFAD/cMunuLv5sVvuExE5zstzZtovSxamvesH/BEYBBwHXA58C7gZZwiR/wAjVPVk4FfAXe529wP9RGQC8BhwnapWRjnGDOBf7j6KcIclEZHjgW/jjCQ6GKgBvuNukwZ84I4wugSYoaovAiuA76jqYFXd65bd7pb7qxu3MU2uXY0NZUwEX6jqJwAisgZ4Q1VVRD4BeuOMcvqEiOTjDPUcHLMrICJXAR8DD6rquw0cYwRwsbvdayKyw11+Ds6Ipcvd8X5ScOabAGcuhVnu86eBSCP+BgXXrQwex5imZsnCtHdVIc8DIa8DOH8fvwEWq+oEcSZaeiukfD6wh8b1IUQahE2AJ1R1+mFuHxSMuQb7mzYesWYoYxqWBWxyn18VXCgiWTjNVyOALm5/QjRv4zYvicj5QCd3+RvApSKS667rLCJHu+t8OCPngtM09i/3+W6ceceNaVaWLIxp2P8Dfici7wL+kOV/AP6iqp8D1wB3Bz/0I7gdGCEiH+DMQfAVgKquBX6BM03mx8BCnOHEASqAE0VkJXA2cIe7/HHgb2Ed3MZ4zoYoN6YFEpE9qpoe7ziMCbKahTHGmJisZmFMExGRq4GfhC1+V1V/FI94jGlKliyMMcbEZM1QxhhjYrJkYYwxJiZLFsYYY2KyZGGMMSYmSxbGGGNi+v+pGOX9+zUqXAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [2, 3, 4], 'min_child_weight': [0.5, 1, 1.5]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [2,3,4]\n",
    "min_child_weight = [0.5,1,1.5]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59868, std: 0.00338, params: {'max_depth': 2, 'min_child_weight': 0.5},\n",
       "  mean: -0.59878, std: 0.00344, params: {'max_depth': 2, 'min_child_weight': 1},\n",
       "  mean: -0.59878, std: 0.00328, params: {'max_depth': 2, 'min_child_weight': 1.5},\n",
       "  mean: -0.58983, std: 0.00412, params: {'max_depth': 3, 'min_child_weight': 0.5},\n",
       "  mean: -0.59040, std: 0.00387, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59046, std: 0.00379, params: {'max_depth': 3, 'min_child_weight': 1.5},\n",
       "  mean: -0.58859, std: 0.00456, params: {'max_depth': 4, 'min_child_weight': 0.5},\n",
       "  mean: -0.58805, std: 0.00479, params: {'max_depth': 4, 'min_child_weight': 1},\n",
       "  mean: -0.58802, std: 0.00414, params: {'max_depth': 4, 'min_child_weight': 1.5}],\n",
       " {'max_depth': 4, 'min_child_weight': 1.5},\n",
       " -0.5880184042667097)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=610,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=3,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([181.93013234, 176.43194532, 168.38340478, 239.77920609,\n",
       "        240.83822436, 239.22620106, 310.62270141, 309.15178475,\n",
       "        254.80762434]),\n",
       " 'std_fit_time': array([0.60003776, 6.71866648, 0.75646478, 1.20052369, 0.97672223,\n",
       "        1.21609433, 0.70367982, 3.48783017, 0.45812844]),\n",
       " 'mean_score_time': array([0.96912003, 1.00170135, 0.97601218, 1.23970518, 1.67980475,\n",
       "        1.49172077, 1.80803275, 2.57088966, 1.42685742]),\n",
       " 'std_score_time': array([0.18220738, 0.2623379 , 0.13071643, 0.33265395, 0.67359498,\n",
       "        0.3942124 , 0.39900972, 1.15028698, 0.21700276]),\n",
       " 'param_max_depth': masked_array(data=[2, 2, 2, 3, 3, 3, 4, 4, 4],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[0.5, 1, 1.5, 0.5, 1, 1.5, 0.5, 1, 1.5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 2, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 2, 'min_child_weight': 1},\n",
       "  {'max_depth': 2, 'min_child_weight': 1.5},\n",
       "  {'max_depth': 3, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 1.5},\n",
       "  {'max_depth': 4, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 4, 'min_child_weight': 1},\n",
       "  {'max_depth': 4, 'min_child_weight': 1.5}],\n",
       " 'split0_test_score': array([-0.59332226, -0.59304087, -0.59340887, -0.58284721, -0.58391306,\n",
       "        -0.58442552, -0.58083895, -0.57948129, -0.58097904]),\n",
       " 'split1_test_score': array([-0.59700954, -0.59703077, -0.5971598 , -0.58815   , -0.58883351,\n",
       "        -0.58823015, -0.58678139, -0.58643152, -0.58594601]),\n",
       " 'split2_test_score': array([-0.59855699, -0.59956615, -0.59911324, -0.59036775, -0.59069691,\n",
       "        -0.59096111, -0.58996569, -0.58986089, -0.59024518]),\n",
       " 'split3_test_score': array([-0.60241595, -0.60224617, -0.60231929, -0.5936297 , -0.59402748,\n",
       "        -0.59421485, -0.59112048, -0.59219246, -0.59019447]),\n",
       " 'split4_test_score': array([-0.60207152, -0.60202512, -0.6018845 , -0.59414609, -0.59453494,\n",
       "        -0.59445384, -0.59426067, -0.59230923, -0.59272876]),\n",
       " 'mean_test_score': array([-0.59867505, -0.59878162, -0.59877695, -0.58982789, -0.59040093,\n",
       "        -0.59045685, -0.58859309, -0.58805482, -0.5880184 ]),\n",
       " 'std_test_score': array([0.00337551, 0.0034398 , 0.00327872, 0.00412157, 0.00386984,\n",
       "        0.00378649, 0.00455629, 0.00478798, 0.0041407 ]),\n",
       " 'rank_test_score': array([7, 9, 8, 4, 5, 6, 3, 2, 1], dtype=int32),\n",
       " 'split0_train_score': array([-0.57787095, -0.57770554, -0.57785196, -0.5375009 , -0.53813077,\n",
       "        -0.53854465, -0.48431133, -0.48563716, -0.48874122]),\n",
       " 'split1_train_score': array([-0.57602234, -0.57609131, -0.57636062, -0.5360284 , -0.53643072,\n",
       "        -0.53706422, -0.48174679, -0.48409291, -0.48545493]),\n",
       " 'split2_train_score': array([-0.57602754, -0.57593957, -0.57603694, -0.53657685, -0.53692615,\n",
       "        -0.53774652, -0.48249182, -0.48441987, -0.4861241 ]),\n",
       " 'split3_train_score': array([-0.57567166, -0.57554337, -0.57558225, -0.53569914, -0.53633961,\n",
       "        -0.53681185, -0.48260191, -0.48350283, -0.48646102]),\n",
       " 'split4_train_score': array([-0.57492299, -0.57520843, -0.57522205, -0.53567339, -0.5363129 ,\n",
       "        -0.53673853, -0.48145994, -0.48310165, -0.48466359]),\n",
       " 'mean_train_score': array([-0.57610309, -0.57609765, -0.57621076, -0.53629574, -0.53682803,\n",
       "        -0.53738116, -0.48252236, -0.48415088, -0.48628897]),\n",
       " 'std_train_score': array([0.00097122, 0.00086119, 0.00090761, 0.00068501, 0.00068826,\n",
       "        0.00068187, 0.00099377, 0.00087242, 0.0013717 ])}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588018 using {'max_depth': 4, 'min_child_weight': 1.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAELCAYAAADgPECFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XucVXW9//HXZy7MiFzkqiASXtBQhIEmlQzvN8xQT5jQ5QcpWSdPlp0ITI+X0g55KM1fDzXT1LQ000r0hwZykUyRMxiIoslFJRpEQQ2Q68z+/P5Y3z2zZ2bvPXvPzJo9wPtZ+7Fu3+93ffZyz/6wvmvt9TV3R0REJC5FhQ5ARET2bko0IiISKyUaERGJlRKNiIjESolGRERipUQjIiKxUqIREZFYKdGIiEislGhERCRWJYUOoCPo3bu3Dxo0qNBhiIjsUZYsWbLR3fs0V06JBhg0aBBVVVWFDkNEZI9iZm/nUk5dZyIiEislGhERiZUSjYiIxEqJRkREYlWwRGNmPc1sjpmtDNMeGcrVmtnS8JqZsv50M3sprH/OzI4I6wea2Xwz+5uZvWxm57bXexIRkaYKeUYzDZjr7oOBuWE5ne3uXhFeY1PW3wF80d0rgN8C14T11wCPuPsIYDxwezzhi4hILgqZaM4H7g/z9wMX5FnfgW5hvjtQ3cx6EREpgEL+juZAd18P4O7rzaxvhnLlZlYF1ADT3f1PYf1kYJaZbQc2AyeE9dcDs83sm8D+wBlxvQERaTvujuO4OwkS4ODUr0tOARKeaLCtQX2chCeatOnUl0uQqF+XYT8Npqn7Ce03qNuoXLppajxN6jcql/BEgzYbl2twfNLEWHd8UmJuEmNYHtJrCMP7DI/1v22sicbMngEOSrPp6jyaGeju1WZ2GDDPzJa7+2rgSuBcd3/RzKYAPyVKPhOA+9z9J2Y2CnjAzIa6h09efWyXAZcBDBw4MP83J9JCu2p3sXnX5ui1czNbdm1h86766eadm9mye0vdtDZR27ovujRfxnVfxBm+EFO/qJr9Mk5Tri4+mn7ZZ9qnFMYlQy/ZsxONu2c8mzCzDWbWL5zN9APezdBGdZiuMbMFwAgz2wwMd/cXQ7HfAU+H+UuBc0KdF8ysHOjduH13vwu4C6CyslKfdMmZu7OtZhubd26uTxjJRLFzc8OkkbI+uW5H7Y6s7ZcXl9OtUze6dupK105dKS4qxjCKioowDDOj7n9hPvp/9L8iK2qyrq5O42lqOauvD6QtD9S1b2YZy2WLsS4+yFo+dZq6nyKKMu67QXypdVPar3t/ad5bavvNHsMMMTY5/s28t8bHItt/v8bHP125Iooy7jvdf7/OJZ1b/seQo0J2nc0EJgLTw/TxxgXCnWjb3H2nmfUGTgRuBj4AupvZke7+BnAm8FqothY4HbjPzIYA5cB7cb8Z2bPUJGoankns3Mzm3ZubJoo0Zxxbdm2h1msztm0YXTp1oVunbnWvQ7sfSreybnUJJLm+a6eudCtruK5Tcad2PBIi8StkopkOPGJmlxIlh4sAzKwS+Lq7TwaGAL8wswTRjQvT3X1FKPdV4LGw7QPgktDufwK/NLMriW4MmOTJfgPZa7g7O2p3pO96SumWapwgkuu31WzL2n5pUWmDRHBA+QEM7DawQfJITRB1iaKsG11Ku9T9q1lEwPQdHHWd6aGa7S/hCbbs2pL+GkXjpJGma2p3YnfW9vcv3b9pIkhJHqlJIzVRdO3UlfLi8rquBRFJz8yWuHtlc+X09GZpldQL22mvUaRe2G6UPLbu2pr1QnCxFTdJFP3275c+aZR2a9A11bVTV0qK9PEW6Qj0l7iPy+fCdrozjnwubHcr60bfzn05/IDD688eSpteo0hu61zSWWcVInsBJZq9QPLCdmoiSF7YTnf3UxwXtht3PenCtogkKdF0AMkL25luj0133SI1UXy0+6Os7ZcUlTQ4U0h3YTv1uoUubItIW1KiaSMJT7B199b03U2NuqXS/baiuQvbnUs6N0gEB3c5mCGdhujCtoh0eEo0rbBw3UJ+9OKP2uTCdmpy0IVtEdmb6NurFXqW92RE3xFZr1HowraI7OuUaFphaO+h/Pfo/y50GCIiHZqu8oqISKyUaEREJFZKNCIiEislGhERiZUSjYiIxEqJRkREYqVEIyIisVKiERGRWCnRiIhIrJRoREQkVko0IiISKyUaERGJlRKNiIjESolGRERipUQjIiKxUqIREZFYKdGIiEislGhERCRWSjQiIhKrkkIHsEfb/gH865+wX4/oVbofmBU6KhGRDqUgicbMegK/AwYBbwGfd/cP0pSrBZaHxbXuPjasPw2YAXQClgCXunuNmRnwM+BcYBswyd1fiu2NrFkAv59Uv1zcqT7plB9QP79f6nzqtjAt7w5FxbGFKSJSSIU6o5kGzHX36WY2LSxPTVNuu7tXpK4wsyLgfuB0d3/DzH4ATATuAcYAg8PreOCOMI3HIcfD538dndls/wC2f5gy/wFsXgcbXonmd23N3lZ59+wJqrxRskpuK90vtrcnItIWCpVozgdOCfP3AwtIn2jS6QXsdPc3wvIc4CqiRHM+8Gt3d2CRmR1gZv3cfX1bBd5At/5w9Pm5la3dXZ+IdjRKSKkJKrntX/+o3+a1mdstKc9y9nRA5m1l3aFIl+hEJH6FSjQHJr/83X29mfXNUK7czKqAGmC6u/8J2AiUmlmlu1cB44BDQvmDgX+k1F8X1sWTaPJRXApd+kSvfLjDzi1pElSjZLXjw2jdh2th/cvRut0fZWnY6s+icj17Sm4rLW/VoRCRfUtsicbMngEOSrPp6jyaGeju1WZ2GDDPzJa7+2ozGw/cYmZlwGyiRASQ7kq8Z4jvMuAygIEDB+YRUjszg/Ju0YuP5Ve3ZmeUfDIlqMZnVh+8Xb/eE5nbLdkvv7OnZIIq66azKJF9UGyJxt3PyLTNzDYku7TMrB/wboY2qsN0jZktAEYAq939BWB0aOss4MhQZR31ZzcAA4DqDG3fBdwFUFlZmTYZ7fFKyqDrgdErH4kE7NrSzNlTcv2H8P6b9dtqtmdu14pCMsrj7Cm5rqSsdcdCRAqmUF1nM4ku4E8P08cbFzCzHsA2d99pZr2BE4Gbw7a+7v5uOKOZCtyU0u5/mNnDRDcB/Cu26zN7s6KiqFutvDv0GJRf3d07GiWiDGdP2z+Ebe/DptX13X7pTz4jpZ0bnSFl6PZrnLzKuuqWc5ECK1SimQ48YmaXAmuBiwDMrBL4urtPBoYAvzCzBNEPS6e7+4pQf4qZnRfW3+Hu88L6WUS3Nq8iur35K+31hiQoLYfSg6Brul7TLBIJ2PmvLAnqw4brkwlq2/tQuzNzu1accht5LmdPKd2AJZ1adyxEBACLbtDat1VWVnpVVVWhw5CW2r09S/deljOrHf/K3m6nLilJ6IDmz56S2zp10VmU7BPMbIm7VzZXTk8GkD1f6X7Rq1u//OolaqNkk0xGO1KTUprEtfGN+uXaXZnbLSrJ7+wpdX2x/iRl76NPtey7ioqhc8/olQ/3cBaV49nT1nfgvddg+7+i7sFsOnVNSUChC6+4NLqRAoumdS/q55tsS1mGzNsa1LUs24qalsm2zyb7zdZuo/1mbDfdftO1m2F7tvfSXLs6Q20VJRqRfJlBp87Rq/vB+dWtrYnOonL50e72D6KzqERNdLu5J6Ik516/TMp83fYwzbgtpZ7kIZcE1lyyziOpNpvIjSZJuCX7PPJsGPpvsR45JRqR9lRcAvv3il4dQWpiSk1KDZJUozLNJrB0ddO1S5ZteSTNJvvNts9GybpN3kvjcq3YZzL557TPRHQTje9ufp8Z/5smoO/HY/+YKdGI7MvMwPRAV4mXfqYtIiKxUqIREZFYKdGIiEislGhERCRWSjQiIhIrJRoREYmVEo2IiMRKiUZERGKlRCMiIrFSohERkVgp0YiISKyUaEREJFZKNCIiEislGhERiZUSjYiIxEqJRkREYqVEIyIisVKiERGRWCnRiIhIrJRoREQkVko0IiISKyUaERGJlRKNiIjEqiCJxsx6mtkcM1sZpj0ylKs1s6XhNTNl/Wlm9pKZvWJm95tZSVj/RTN7ObyeN7Ph7fWeREQkvUKd0UwD5rr7YGBuWE5nu7tXhNdYADMrAu4Hxrv7UOBtYGIo/yZwsrsPA34I3BXnmxARkeYVKtGcT5QsCNML8qjbC9jp7m+E5TnA5wDc/Xl3/yCsXwQMaINYRUSkFQqVaA509/UAYdo3Q7lyM6sys0VmlkxGG4FSM6sMy+OAQ9LUvRR4KlMAZnZZaLvqvffea9m7EBGRZpXE1bCZPQMclGbT1Xk0M9Ddq83sMGCemS1399VmNh64xczKgNlATaN9n0qUaD6dqWF3v4vQtVZZWel5xCQSq927d7Nu3Tp27NhR6FBEACgvL2fAgAGUlpa2qH5eiSZctD/E3V9urqy7n5GlnQ1m1s/d15tZP+DdDG1Uh+kaM1sAjABWu/sLwOjQ1lnAkSltDwPuBsa4+6ac35xIB7Fu3Tq6du3KoEGDMLNChyP7OHdn06ZNrFu3jkMPPbRFbTTbdWZmC8ysm5n1BJYB95rZT1u0t3ozqb+APxF4PM1+e4QzFsysN3AisCIs9w3TMmAqcGdYHgj8AfhyyjUckT3Kjh076NWrl5KMdAhmRq9evVp1hp3LNZru7r4Z+DfgXnf/BJDxbCVH04EzzWwlcGZYxswqzezuUGYIUGVmy4D5wHR3XxG2TTGz14CXgSfcfV5Yfy3RzQK3h1uiq1oZp0hBKMlIR9Laz2MuXWcloXvr8+R3fSWj0KV1epr1VcDkMP88cGyG+lOAKWnWT07WFxGRjiGXM5ofAH8GVrn7/4YL8yvjDUtECuXDDz/k9ttvb1HdW2+9lW3btrU6hqqqKq644opWt5M0adIkHn300Sbrq6urGTduHAALFizgvPPOS1t/0KBBbNy4sc3iSZo8eTIrVqzIWiZT7G+99Ra//e1vM9Z7+umnOeqoozjiiCOYPn162jL33Xcfffr0oaKigoqKCu6+++605Vqr2UTj7r9392Hu/o2wvMbdPxdLNCJScB0h0VRWVnLbbbe1up3m9O/fP+2XeHu5++67Ofroo1tUN1uiqa2t5fLLL+epp55ixYoVPPTQQxkT2sUXX8zSpUtZunQpkyfH0yHUbNeZmd0M3AhsB54GhgPfdvcHY4lIROrc8MSrrKje3KZtHt2/G9d99piM26dNm8bq1aupqKjgzDPPpG/fvjzyyCPs3LmTCy+8kBtuuIGPPvqIz3/+86xbt47a2lr+67/+iw0bNlBdXc2pp55K7969mT9/ftr2u3TpwuWXX84zzzxDjx49+NGPfsT3vvc91q5dy6233srYsWNZsGABM2bM4Mknn+T6669n7dq1rFmzhrVr1/Ltb38769nOr3/9a2bMmIGZMWzYMB544AEAFi5cyE9/+lPeeecdbr75ZsaNG8dbb73FeeedxyuvvNKgjU2bNjFhwgTee+89jjvuONwz/wLi5ptvpry8nCuuuIIrr7ySZcuWMW/ePObOncu9997Lgw8+yOzZs7nuuuvYuXMnhx9+OPfeey9dunThlFNOYcaMGVRWVnLPPffw4x//mP79+zN48GDKysr4+c9/njH2adOm8dprr1FRUcHEiRO58sor62JavHgxRxxxBIcddhgA48eP5/HHH29xUmutXLrOzgo3A5wHrCO6lbjJ9RER2TtMnz6dww8/nKVLl3LmmWeycuVKFi9ezNKlS1myZAkLFy7k6aefpn///ixbtoxXXnmFc845hyuuuIL+/fszf/78jEkG4KOPPuKUU05hyZIldO3alWuuuYY5c+bwxz/+kWuvvTZtnddff50///nPLF68mBtuuIHdu3enLffqq69y0003MW/ePJYtW8bPfvazum3r16/nueee48knn2TatExPvYrccMMNfPrTn+Zvf/sbY8eOZe3atRnLnnTSSfzlL38Boi6/rVu3snv3bp577jlGjx7Nxo0bufHGG3nmmWd46aWXqKys5Kc/bXjjbnV1NT/84Q9ZtGgRc+bM4fXXX2+wPV3s06dPZ/To0SxdupQrr7yS6upqzj33XAD++c9/csgh9b9jHzBgAP/85z/Txv/YY48xbNgwxo0bxz/+8Y+sx6WlcrkZIPkLnXOBh9z9fd0RI9I+sp15tIfZs2cze/ZsRowYAcDWrVtZuXIlo0eP5rvf/S5Tp07lvPPOY/To0Tm32alTJ8455xwAjj32WMrKyigtLeXYY4/lrbfeSlvnM5/5DGVlZZSVldG3b182bNjAgAFNnzA1b948xo0bR+/evQHo2bNn3bYLLriAoqIijj76aDZs2JA1xoULF/KHP/yhbt89eqR97i8An/jEJ1iyZAlbtmyhrKyMkSNHUlVVxV/+8hduu+02Fi1axIoVKzjxxBMB2LVrF6NGjWrQxuLFizn55JPr4r3ooot44436X2jkEnv//v2ZNWsWQNozsHTf25/97GeZMGECZWVl3HnnnUycOJF58+Y1KddauSSaJ8zsdaKus2+YWR9AP1kW2Qe4O1dddRVf+9rXmmxbsmQJs2bN4qqrruKss87KeDbSWGlpad2XXlFREWVlZXXzNTU1aeskywAUFxdnLOfuGW/FTW0jW1dYUq7/oC4tLWXQoEHce++9fOpTn2LYsGHMnz+f1atXM2TIEFavXs2ZZ57JQw89lLGN5uLJN/YBAwY0ODtZt24d/fv3b1KuV69edfNf/epXmTp1arNtt0QuNwNMA0YBle6+G/iI6KGYIrIX6tq1K1u2bAHg7LPP5le/+hVbt24Foi6Zd999l+rqajp37syXvvQlvvvd7/LSSy81qVsIp59+Oo888gibNkUPBXn//fdb1M5JJ53Eb37zGwCeeuopPvjgg2bLz5gxg5NOOonRo0dz5513UlFRgZlxwgkn8Ne//pVVq1YBsG3btgZnKwDHHXcczz77LB988AE1NTU89thjzcaY7Vh/8pOfZOXKlbz55pvs2rWLhx9+mLFjxzYpt379+rr5mTNnMmTIkGb32xK53AxQCnwZOClk+GcJv8QXkb1Pr169OPHEExk6dChjxozhC1/4Ql1XT5cuXXjwwQdZtWoVU6ZMoaioiNLSUu644w4ALrvsMsaMGUO/fv2yXqeJyzHHHMPVV1/NySefTHFxMSNGjOC+++7Lu53rrruOCRMmMHLkSE4++WQGDhyYtfzo0aO56aabGDVqFPvvvz/l5eV13Yl9+vThvvvuY8KECezcuROAG2+8kSOPrHtyFgcffDDf//73Of744+nfvz9HH3003bt3z7rPYcOGUVJSwvDhw5k0aRIXX3wxkydPZtasWZSUlPDzn/+cs88+m9raWi655BKOOSbqhr322muprKxk7Nix3HbbbcycOZOSkhJ69uzZomOVC2vuNCz8Ur+U+sf6fxmoDT+O3CtUVlZ6VZUeIiAdw2uvvRbbvyyl49q6dStdunShpqaGCy+8kEsuuYQLL7yw0GHVSfe5NLMl7l6ZoUqdXK7RfNLdU0eqnBceCyMiIm3k+uuv55lnnmHHjh2cddZZXHBBPsN0dWy5JJpaMzvc3VcDhCcD1MYblojs6Y4//vi6rqKkBx54gGOPTftkqbxs2rSJ009v8hQr5s6d2+ACd1uKe58zZsxodRsdVS6JZgow38zWAAZ8DPhKrFGJyB7vxRdfjK3tXr16sXTp0tja7yj73Fs0m2jcfa6ZDQaOIko0rwMVcQcmIiJ7h5wGPnP3nUSP5AfAzH4PZL8NQ0REhNweQZOOHg0gIiI5aWmiaf6nqSIiImRJNGb2hJnNTPN6gmgUSxHZC3WEYQI0Hk29lo5Hc8kll9C3b1+GDh3a6jhbK9sZzQzgJ2leM4gesCkie6GOkGg0Hk3zmks0kyZN4umnn25paG0q480A7v5sewYiImk8NQ3eWd62bR50LIxJP+IiaDwa2PPHo4Ho+WuZnobd3lp6jUZE9lIaj2bPH4+mo8np9mYRKZAsZx7tQePR7Jnj0XQ0SjQikpHGo2leRxyPpqNptussw91nD5jZt8ysvD2CFJH2o/Fo9vzxaDqaXK7RrAG2Ar8Mr83ABuDIsCwie5HU8WjmzJlTNx7Nsccey7hx49iyZQvLly/nuOOOo6KigptuuolrrrkGqB+P5tRTTy1I7Knj0QwfPpzvfOc7LWrnuuuuY+HChYwcOZLZs2fnNB7N+vXrGTVqFAceeGDG8WiGDRvGCSec0OQaTOp4NGeccUbe49HccsstTa7RTJgwgVGjRvH3v/+dAQMGcM8997ToWLSFXMajWejuJ6VbZ2avunthBzVvAxqPRjoSjUezb9qbx6PJ5Yymj5nVpfMw3zss7sonUBERSe/666+noqKCoUOHcuihh+5z49H8J/Ccma0mesbZocA3zGx/6kfdFBFpQOPR5GefHo/G3WeFYQI+ThgmwN13hM23tmSnZtYT+B0wCHgL+Ly7N7naZma1QPLXamvdfWxYfxrREwo6AUuAS929JqXeJ4FFwMXuXrif/YrswzQejSTlctdZKfA14L+Aa4DJYV1rTAPmuvtgYG5YTme7u1eEVzLJFBGdSY1396HA28DElHiLgR8Df25ljCIi0gZyuUZzB/AJ4Pbw+kRY1xrnU9/tdj+QT2dkL2CnuyfvD5wDfC5l+zeBx4B3WxmjiIi0gVyu0XzS3YenLM8zs2Wt3O+B7r4ewN3Xm1nfDOXKzawKqAGmu/ufgI1AqZlVunsVMA44BMDMDgYuBE4DPtnKGEVEpA3kkmhqzexwd18NYGaHAbXNVTKzZ4CD0my6Oo/4Brp7ddjnPDNb7u6rzWw8cIuZlQGziRIRRNeMprp7bXO/6jWzy4DLgGbvkRcRkZbLpetsCjDfzBaY2bPAPKI70bJy9zPcfWia1+PABjPrBxCmabu53L06TNcAC4ARYfkFdx/t7scBC4GVoUol8LCZvUV0pnO7maXtlnP3u9y90t0r+/Tpk8NhENk3dIRhAjQeTb04x6NZsGAB3bt3p6KigoqKCn7wgx/kHnwemk007j4XGAxcEV5HAa39JM2k/gL+RODxxgXMrEc4Y8HMegMnAivCct8wLQOmAneGWA9190HuPgh4FPhG6G4TkRx1hESj8Wia11bj0SSfAL106dKcn1eXr5wequnuO4GXk8tm9nugNf1N04FHzOxSYC1wUWi3Evi6u08GhgC/MLMEUUKc7u7J1D/FzM4L6+9w93mtiEWkw/rx4h/z+vuvN18wDx/v+XGmHjc143aNR6PxaNpaS8ejye2xphm4+yZ3P93dB4fp+2F9VUgyuPvz7n6suw8P03tS6k9x9yHufpS7p/0tj7tP0m9oRPKn8Wj2rfFoXnjhBYYPH86YMWN49dVX866fi5YOE7DnPadaZA+U7cyjPWg8mr17PJqRI0fy9ttv06VLF2bNmsUFF1zAypUrm6+Yp4yJxsyeIH1CMaLfsojIXk7j0TRvTx6Pplu3bnXz5557Lt/4xjfYuHFjXaJuK9m6zmYAP0nzmgF0zPFCRaTVNB7NvjMezTvvvFOXuBYvXkwikYjlWXEZz2jc/dk235uIdHip49GMGTOmbjwaiC7kP/jgg6xatYopU6ZQVFREaWkpd9wRPSwkOR5Nv379sl6niUvqeDTFxcWMGDGC++67L+92rrvuOiZMmMDIkSM5+eSTcxqP5qabbmLUqFHsv//+GcejST5k9MYbb+TII4+sq586Hk3//v3zHo9m0qRJXHzxxUyePLmu+2zChAksWLCAjRs3MmDAAG644QYuvfRS7rzzTgC+/vWv8+ijj3LHHXdQUlLCfvvtx8MPP5zzmVw+mh2PZl+g8WikI9F4NPumfX08GhERidm+Ph5NHTM7yN3fiSsYEdl7aDya/OzT49E0MgsYGUcgIrJ30Xg0kpRv11nbXyUSkSZ07VQ6ktZ+HvNNNL9s1d5EpFnl5eVs2rRJyUY6BHdn06ZNlJeXt7iNvLrO3L1lT9oTkZwNGDCAdevW8d577xU6FBEg+sdPuicx5Kqlj6ARkZiUlpZy6KGHFjoMkTaj25tFRCRWSjQiIhIrJRoREYmVEo2IiMRKiUZERGKlRCMiIrFSohERkVgp0YiISKyUaEREJFZKNCIiEislGhERiZUSjYiIxEqJRkREYqVEIyIisVKiERGRWCnRiIhIrAqSaMysp5nNMbOVYdojQ7laM1saXjNT1p9mZi+Z2Stmdr+ZlaRsOyWUf9XMnm2P9yMiIpkV6oxmGjDX3QcDc8NyOtvdvSK8xgKYWRFwPzDe3YcCbwMTw7YDgNuBse5+DHBRzO9DRESaUahEcz5RsiBML8ijbi9gp7u/EZbnAJ8L818A/uDuawHc/d02iFVERFqhUInmQHdfDxCmfTOUKzezKjNbZGbJZLQRKDWzyrA8DjgkzB8J9DCzBWa2xMz+T1xvQEREclPSfJGWMbNngIPSbLo6j2YGunu1mR0GzDOz5e6+2szGA7eYWRkwG6gJ5UuATwCnA/sBL5jZopSzn9T4LgMuAxg4cGAeIYmISD5iSzTufkambWa2wcz6uft6M+sHpO3icvfqMF1jZguAEcBqd38BGB3aOovoTAZgHbDR3T8CPjKzhcBwoEmicfe7gLsAKisrvWXvUkREmlOorrOZhAv4Yfp44wJm1iOcsWBmvYETgRVhuW+YlgFTgTtDtceB0WZWYmadgeOB12J8HyIi0oxCJZrpwJlmthI4MyxjZpVmdncoMwSoMrNlwHxguruvCNummNlrwMvAE+4+D8DdXwOeDusXA3e7+yvt9aZERKQpc1evUWVlpVdVVRU6DBGRPYqZLXH3yubK6ckAIiISKyUaERGJlRKNiIjESolGRERipUQjIiKxUqIREZFYKdGIiEislGhERCRWSjQiIhIrJRoREYmVEo2IiMRKiUZERGKlRCMiIrFSohERkVgp0YiISKyUaEREJFZKNCIiEislGhERiZUSjYiIxEqJRkREYqVEIyIisVKiERGRWCnRiIhIrJRoREQkVko0IiISKyUaERGJlRKNiIjESolGRERiVZBEY2Y9zWzA0WKjAAAMiElEQVSOma0M0x4ZytWa2dLwmpmy/jQze8nMXjGz+82sJKzvbmZPmNkyM3vVzL7SXu9JRETSK9QZzTRgrrsPBuaG5XS2u3tFeI0FMLMi4H5gvLsPBd4GJobylwMr3H04cArwEzPrFOP7EBGRZhQq0ZxPlCwI0wvyqNsL2Onub4TlOcDnwrwDXc3MgC7A+0BN68MVEZGWKlSiOdDd1wOEad8M5crNrMrMFplZMhltBErNrDIsjwMOCfM/B4YA1cBy4FvunojlHYiISE5K4mrYzJ4BDkqz6eo8mhno7tVmdhgwz8yWu/tqMxsP3GJmZcBs6s9azgaWAqcBhwNzzOwv7r45TXyXAZcBDBw4MI+QREQkH7ElGnc/I9M2M9tgZv3cfb2Z9QPezdBGdZiuMbMFwAhgtbu/AIwObZ0FHBmqfAWY7u4OrDKzN4GPA4vTtH0XcBdAZWWlt+xdiohIcwrVdTaT+gv4E4HHGxcwsx7hjAUz6w2cCKwIy33DtAyYCtwZqq0FTg/bDgSOAtbE9i5ERKRZhUo004EzzWwlcGZYxswqzezuUGYIUGVmy4D5RGcqK8K2KWb2GvAy8IS7zwvrfwh8ysyWE93NNtXdN7bPWxIRkXQs6mXat1VWVnpVVVWhwxAR2aOY2RJ3r2yunJ4MICIisVKiERGRWCnRiIhIrJRoREQkVko0IiISKyUaERGJVWxPBtgXVL31PncsWE1RkVFsRnFxmBaFlxlFRUZJWC4yo7gIiouKoqlZ3XxdG0WNXmnbqN+WWrbIjJLilDINtof9mlFURPY2wnz0bFIRkdZRommF7btreWfzDmoTTsKdmoSTSDi17iQSUJNIUJsg2labIOFQG7bXJqJXR2ZGk8SZTKa5JMaoDBkTWbbEWNdG432kSbxN9xWSd6P2U9ura6NJvJkTb5OYGhwPKCkqoshQghZpRImmFUYP7sPowX1a1UYiERJUSD4Nk1W0nC6RJRNV3bbahgkuao/6cintJduom09pvzbhTWKqdae2tlEbTeJNvpeU5JrSfq07u3cnGsSbSH0fnn7f6d5zB8/PFBm5JcbGybVx4g3bM3EyH4hsv8POeviy1mv7/WX7wXj2ei3bX6aKLd9XC49JDMcrm2zVLqocwOTRh7Wo3Vwp0RRYUZHRqUj/As6He5RsahIJEgkaJMImSblx0mxJ4m2QGKE2kQhtZP+HQpN/DKQm3kSjeGsbtVE3n8DI8vlowSYjOlvNtDHz/ixzvWxtkq3N7PWyyXbmmK3JTNWy12nbfTVXM/uxbNn+Mv036NUl/rEhlWhkj2OW7CYrLnQoIpID3XUmIiKxUqIREZFYKdGIiEislGhERCRWSjQiIhIrJRoREYmVEo2IiMRKiUZERGJlLX2kwd7EzN4D3m5h9d7AxjYMp6101Lig48amuPKjuPKzN8b1MXdv9jlcSjStZGZV7l5Z6Dga66hxQceNTXHlR3HlZ1+OS11nIiISKyUaERGJlRJN691V6AAy6KhxQceNTXHlR3HlZ5+NS9doREQkVjqjERGRWCnRZGBmh5jZfDN7zcxeNbNvpSljZnabma0ys5fNbGTKtolmtjK8JrZzXF8M8bxsZs+b2fCUbW+Z2XIzW2pmVe0c1ylm9q+w76Vmdm3KtnPM7O/hWE5r57impMT0ipnVmlnPsC2u41VuZovNbFmI64Y0ZcrM7HfhmLxoZoNStl0V1v/dzM5u57i+Y2Yrwudrrpl9LGVbbcqxnNnOcU0ys/dS9j85ZVtcf4+5xHVLSkxvmNmHKdtiOV4p7Reb2d/M7Mk029rv8+XueqV5Af2AkWG+K/AGcHSjMucCTxENencC8GJY3xNYE6Y9wnyPdozrU8n9AWOScYXlt4DeBTpepwBPpqlbDKwGDgM6Acsa140zrkblPwvMa4fjZUCXMF8KvAic0KjMN4A7w/x44Hdh/uhwjMqAQ8OxK27HuE4FOof5f0/GFZa3tvWxyiOuScDP09SN8++x2bgalf8m8Ku4j1dK+98Bfpvh767dPl86o8nA3de7+0thfgvwGnBwo2LnA7/2yCLgADPrB5wNzHH39939A2AOcE57xeXuz4f9AiwCBrTFvlsbVxbHAavcfY277wIeJjq2hYhrAvBQW+y7mbjc3beGxdLwanzB9Hzg/jD/KHC6mVlY/7C773T3N4FVRMewXeJy9/nuvi0sttfnK5fjlUmcf4/5xtUuny8AMxsAfAa4O0ORdvt8KdHkIJxSjiD610qqg4F/pCyvC+syrW+vuFJdSnTWleTAbDNbYmaXtXVMOcQ1KnQzPGVmx4R1HeJ4mVlnoi+gx1JWx3a8QrfGUuBdoi/CjJ8vd68B/gX0IubjlUNcqRp/vsrNrMrMFpnZBW0VUx5xfS506T1qZoeEdR3ieIUuxkOBeSmrYztewK3A94BEhu3t9vlSommGmXUh+uL5trtvbrw5TRXPsr694kqWOZXoi2BqyuoT3X0kUZfa5WZ2UjvG9RLRIyuGA/8X+FOyWpqm2v14EXWb/dXd309ZF9vxcvdad68gOiM4zsyGNg47XbUs69srrig4sy8BlcD/pKwe6NGvzL8A3Gpmh7djXE8Ag9x9GPAM9f9a7xDHi6h76lF3r01ZF8vxMrPzgHfdfUm2YmnWxfL5UqLJwsxKib6cfuPuf0hTZB1wSMryAKA6y/r2igszG0Z0yny+u29Krnf36jB9F/gjbdTlkktc7r452c3g7rOAUjPrTQc4XsF4GnVrxHm8UvbxIbCApt05dcfFzEqA7sD7xHy8cogLMzsDuBoY6+47U+okj9eaUHdEe8Xl7ptSYvkl8IkwX/DjFWT7fLX18ToRGGtmbxF1RZ9mZg82KtN+n6/WXODZm19EWf3XwK1ZynyGhjcDLA7rewJvEl147BHme7ZjXAOJ+lU/1Wj9/kDXlPnngXPaMa6DqP/t1nHA2lCvhOgC7aHU3wxwTHvFFcol/8j2b6fj1Qc4IMzvB/wFOK9RmctpeLH2kTB/DA0v1q6h7W4GyCWuEUQXiAc3Wt8DKAvzvYGVtN1NHbnE1S9l/kJgUZiP8++x2bjCtqOIbiyx9jhejfZ9CulvBmi3z1cJksmJwJeB5aH/FeD7RF/iuPudwCyiO89WAduAr4Rt75vZD4H/DfV+4A27Y+KO61qivtbbo2t71Hh0en4g8MewrgT4rbs/3Y5xjQP+3cxqgO3AeI8+2TVm9h/An4nuQPuVu7/ajnFB9MU0290/Sqkb5/HqB9xvZsVEPQuPuPuTZvYDoMrdZwL3AA+Y2SqiJDg+xPyqmT0CrABqgMu9YXdM3HH9D9AF+H04NmvdfSwwBPiFmSVC3enuvqId47rCzMYSHZP3ie5Ci/vvMZe4ILoJ4OHweU+K83ilVajPl54MICIisdI1GhERiZUSjYiIxEqJRkREYqVEIyIisVKiERGRWCnRiIhIrJRoRPYgFg1b0LuFdSeZWf+2aEskH0o0IvuOSUD/5gqJtDUlGpEWMLNBZva6md1t0WBpvzGzM8zsrxYNrnVceD0fBp563syOCnW/Y2a/CvPHhvqdM+ynl5nNDm38gpQHHprZlywadGupmf0i/DodM9tqZj8xs5csGpisj5mNI3oA5m9C+f1CM98M5Zab2cfjPGay71KiEWm5I4CfAcOAjxM9gffTwHeJHnPzOnCSu48geizQj0K9W4EjzOxC4F7ga14/vktj1wHPhTZmEh6dY2ZDgIuJni5dAdQCXwx19gde8uip088C17n7o0AV8EV3r3D37aHsxlDujhC3SJvTs85EWu5Nd18OYGavAnPd3c1sOTCI6EGd95vZYKLHrJcCuHvCzCYBLwO/cPe/ZtnHScC/hXr/z8ySA9qdTvR04v8Nzxvbj2g8FIjGH/ldmH8QyPbE6uS2Jcn9iLQ1JRqRltuZMp9IWU4Q/W39EJjv7hdaNOjagpTyg4Gt5HbNJN0DCQ24392vamH9pGTMtej7QGKirjOR+HQH/hnmJyVXmll3oi63k4Be4fpJJgsJXWJmNobo0fIAc4FxZtY3bOsZRnCE6O862eYXgOfC/Bagayvej0iLKNGIxOdm4L/N7K9Ewx8k3QLc7u5vEI2AOj2ZMNK4ATjJzF4CziIaw4fwOPlriIaZfhmYQ/TIeoCPgGPMbAlwGvCDsP4+4M5GNwOIxE7DBIjsZcxsq7t3KXQcIkk6oxERkVjpjEakAzCzrwDfarT6r+5+eSHiEWlLSjQiIhIrdZ2JiEislGhERCRWSjQiIhIrJRoREYmVEo2IiMTq/wPEW2vKazioJAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(min_child_weight), len(max_depth))\n",
    "train_scores = np.array(train_means).reshape(len(min_child_weight), len(max_depth))\n",
    "\n",
    "for i, value in enumerate(min_child_weight):\n",
    "    pyplot.plot(max_depth, test_scores[i], label= 'test_min_child_weight:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( '- Log Loss' )\n",
    "pyplot.savefig( 'max_depth_vs_min_child_weght2.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 3\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print (cvresult)\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds4_2_3_610.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators4_2_3_610.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print (logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'StratifiedKFold' object is not iterable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-56-c67f53cbb9ce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m         seed=3)\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0mmodelfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxgb2_3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv_folds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkfold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-55-560945955071>\u001b[0m in \u001b[0;36mmodelfit\u001b[0;34m(alg, X_train, y_train, useTrainCV, cv_folds, early_stopping_rounds)\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n\u001b[0;32m---> 10\u001b[0;31m                          metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mn_estimators\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcvresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/xgboost/training.py\u001b[0m in \u001b[0;36mcv\u001b[0;34m(params, dtrain, num_boost_round, nfold, stratified, folds, metrics, obj, feval, maximize, early_stopping_rounds, fpreproc, as_pandas, verbose_eval, show_stdv, seed, callbacks)\u001b[0m\n\u001b[1;32m    369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    370\u001b[0m     \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 371\u001b[0;31m     \u001b[0mcvfolds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmknfold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnfold\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfpreproc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstratified\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfolds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    372\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    373\u001b[0m     \u001b[0;31m# setup callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/xgboost/training.py\u001b[0m in \u001b[0;36mmknfold\u001b[0;34m(dall, nfold, param, seed, evals, fpreproc, stratified, folds)\u001b[0m\n\u001b[1;32m    236\u001b[0m         \u001b[0midset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mrandidx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mkstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandidx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mkstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnfold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mfolds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 238\u001b[0;31m         \u001b[0midset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfolds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    239\u001b[0m         \u001b[0mnfold\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    240\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'StratifiedKFold' object is not iterable"
     ]
    }
   ],
   "source": [
    "#调整max_depth和min_child_weight之后再次调整n_estimators(6,4)\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=610,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=4,\n",
    "        min_child_weight=1.5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb2_3, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "File b'my_preds4_2_3_610.csv' does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-dc44257686e6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcvresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'my_preds4_2_3_610.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mcvresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcvresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# plot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtest_means\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcvresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'test-mlogloss-mean'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mfrom_csv\u001b[0;34m(cls, path, header, sep, index_col, parse_dates, encoding, tupleize_cols, infer_datetime_format)\u001b[0m\n\u001b[1;32m   1577\u001b[0m                           \u001b[0mparse_dates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_dates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1578\u001b[0m                           \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtupleize_cols\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtupleize_cols\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1579\u001b[0;31m                           infer_datetime_format=infer_datetime_format)\n\u001b[0m\u001b[1;32m   1580\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1581\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mto_sparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'block'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    439\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    442\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    785\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 787\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    788\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    789\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1014\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1015\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1706\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1708\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1710\u001b[0m         \u001b[0mpassed_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: File b'my_preds4_2_3_610.csv' does not exist"
     ]
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('my_preds4_2_3_610.csv')\n",
    "\n",
    "cvresult = cvresult.iloc[100:]\n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(100,cvresult.shape[0]+100)\n",
    "        \n",
    "fig = pyplot.figure(figsize=(10, 10), dpi=100)\n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators_detail4_2_3_610.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第四步：调整树的参数：subsample 和 colsample_bytree\n",
    "(粗调，参数的步长为0.1；下一步是在粗调最佳参数周围，将步长降为0.05，进行精细调整)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8],\n",
       " 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58724, std: 0.00375, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.58651, std: 0.00402, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58374, std: 0.00348, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58382, std: 0.00404, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58250, std: 0.00361, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58257, std: 0.00431, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58847, std: 0.00363, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58609, std: 0.00343, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58440, std: 0.00374, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58394, std: 0.00400, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58319, std: 0.00382, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58236, std: 0.00409, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58802, std: 0.00414, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58538, std: 0.00368, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58476, std: 0.00353, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58394, std: 0.00401, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58282, std: 0.00424, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58223, std: 0.00433, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58889, std: 0.00337, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58659, std: 0.00307, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58521, std: 0.00362, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58354, std: 0.00362, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58263, std: 0.00355, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58199, std: 0.00376, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       " -0.5819852733047861)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=610,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=1.5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([244.48653197, 284.95804029, 304.25084772, 301.86473336,\n",
       "        290.65782981, 277.72902322, 281.34137397, 322.0988028 ,\n",
       "        346.74346795, 346.52571917, 329.0932909 , 320.2234478 ,\n",
       "        316.58118224, 361.58885479, 390.86984301, 390.53020568,\n",
       "        368.73861914, 358.35557027, 348.61038175, 405.94161177,\n",
       "        437.83150415, 434.08740988, 412.87179465, 365.56391244]),\n",
       " 'std_fit_time': array([ 0.58932132,  1.9242042 ,  0.58791157,  4.16445993,  1.09712631,\n",
       "         0.35900611,  1.51602098,  0.64585055,  1.06017441,  3.86949101,\n",
       "         0.81851591,  3.34218712,  0.41922444,  0.5210632 ,  2.43769514,\n",
       "         1.23785982,  0.82658865,  6.70926611,  0.96920294,  1.62428481,\n",
       "         2.43488157,  0.71664801,  5.48040653, 15.63665239]),\n",
       " 'mean_score_time': array([2.73609223, 1.54727759, 1.67018757, 1.50396023, 1.43062925,\n",
       "        1.39851809, 1.39793558, 1.36687064, 1.36167598, 1.31499348,\n",
       "        1.44352536, 1.35663681, 1.37905731, 1.30618839, 1.23293638,\n",
       "        1.30494313, 1.50420084, 1.26892614, 1.56386142, 1.3155611 ,\n",
       "        1.33629527, 1.28031535, 1.28929362, 0.89093103]),\n",
       " 'std_score_time': array([0.09768144, 0.151219  , 0.12101915, 0.07345565, 0.29482357,\n",
       "        0.17873659, 0.14030255, 0.1479569 , 0.1644433 , 0.08670418,\n",
       "        0.1807471 , 0.16324497, 0.16968807, 0.10426725, 0.01541644,\n",
       "        0.11207707, 0.31467632, 0.02786356, 0.37198525, 0.06173432,\n",
       "        0.09032358, 0.0626677 , 0.11081796, 0.05552764]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.7,\n",
       "                    0.7, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.9, 0.9, 0.9, 0.9,\n",
       "                    0.9, 0.9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6, 0.7,\n",
       "                    0.8, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6,\n",
       "                    0.7, 0.8],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " 'split0_test_score': array([-0.58205422, -0.58045313, -0.57805392, -0.57690092, -0.57656019,\n",
       "        -0.57533299, -0.58185415, -0.57994659, -0.57803633, -0.57694146,\n",
       "        -0.57669329, -0.5753527 , -0.58097904, -0.57899001, -0.57854162,\n",
       "        -0.5771532 , -0.57539432, -0.57481347, -0.5826664 , -0.58112655,\n",
       "        -0.57949116, -0.5769049 , -0.57655309, -0.57555658]),\n",
       " 'split1_test_score': array([-0.58432256, -0.5844126 , -0.58244713, -0.58231884, -0.58141833,\n",
       "        -0.58156574, -0.58749598, -0.5857521 , -0.58302   , -0.58266491,\n",
       "        -0.58165425, -0.5812971 , -0.58594601, -0.58422565, -0.58501553,\n",
       "        -0.58265074, -0.58167364, -0.58083472, -0.59092308, -0.58556015,\n",
       "        -0.58263935, -0.58274979, -0.58148542, -0.58070429]),\n",
       " 'split2_test_score': array([-0.58701993, -0.58597817, -0.58413659, -0.58438357, -0.58196066,\n",
       "        -0.58193529, -0.59054955, -0.58838647, -0.58544427, -0.58518415,\n",
       "        -0.58382464, -0.58230014, -0.59024518, -0.58580486, -0.58409989,\n",
       "        -0.58442743, -0.58339826, -0.58255789, -0.58795198, -0.58758796,\n",
       "        -0.58658921, -0.58484128, -0.58301072, -0.58237665]),\n",
       " 'split3_test_score': array([-0.59144405, -0.5898552 , -0.58548706, -0.58738213, -0.58624419,\n",
       "        -0.58704196, -0.59026095, -0.58631392, -0.58639152, -0.5862048 ,\n",
       "        -0.58636519, -0.58586612, -0.59019447, -0.58894775, -0.5873326 ,\n",
       "        -0.58638155, -0.58615779, -0.58579324, -0.59153584, -0.58938252,\n",
       "        -0.58909613, -0.58707589, -0.5852685 , -0.58497494]),\n",
       " 'split4_test_score': array([-0.59136996, -0.59184141, -0.58858509, -0.58810296, -0.58629782,\n",
       "        -0.586959  , -0.5921792 , -0.59003603, -0.58913056, -0.58869344,\n",
       "        -0.58741052, -0.58696466, -0.59272876, -0.58894525, -0.58881643,\n",
       "        -0.58906929, -0.58750205, -0.58713838, -0.59137858, -0.58928727,\n",
       "        -0.58821399, -0.58611238, -0.58684787, -0.58631523]),\n",
       " 'mean_test_score': array([-0.58724189, -0.58650778, -0.58374166, -0.58381742, -0.58249601,\n",
       "        -0.58256673, -0.58846774, -0.58608678, -0.58440425, -0.58393746,\n",
       "        -0.58318932, -0.58235586, -0.5880184 , -0.58538249, -0.58476097,\n",
       "        -0.58393613, -0.58282493, -0.58222724, -0.58889103, -0.58658873,\n",
       "        -0.58520579, -0.58353669, -0.58263286, -0.58198527]),\n",
       " 'std_test_score': array([0.00374651, 0.00402438, 0.00348379, 0.00403675, 0.00361065,\n",
       "        0.00431387, 0.00363341, 0.00342781, 0.00373719, 0.00399689,\n",
       "        0.00381739, 0.00409218, 0.0041407 , 0.00368199, 0.0035282 ,\n",
       "        0.0040057 , 0.00423911, 0.00433235, 0.00337466, 0.00306528,\n",
       "        0.00361504, 0.00361839, 0.00355343, 0.00376317]),\n",
       " 'rank_test_score': array([21, 19, 10, 11,  4,  5, 23, 18, 14, 13,  8,  3, 22, 17, 15, 12,  7,\n",
       "         2, 24, 20, 16,  9,  6,  1], dtype=int32),\n",
       " 'split0_train_score': array([-0.49376095, -0.48978335, -0.48778302, -0.48852351, -0.48765037,\n",
       "        -0.48896529, -0.49003464, -0.48791281, -0.48539918, -0.48563823,\n",
       "        -0.48596089, -0.48588008, -0.48874122, -0.48659664, -0.48340284,\n",
       "        -0.48291721, -0.48468668, -0.48531159, -0.48640451, -0.48263076,\n",
       "        -0.4809351 , -0.48014232, -0.48146996, -0.48277788]),\n",
       " 'split1_train_score': array([-0.49134931, -0.48850821, -0.48621584, -0.48712599, -0.48670902,\n",
       "        -0.48849719, -0.48888596, -0.48550637, -0.48437009, -0.48398248,\n",
       "        -0.48434373, -0.48543083, -0.48545493, -0.48303334, -0.481823  ,\n",
       "        -0.48044818, -0.48245404, -0.48400139, -0.48321381, -0.48151884,\n",
       "        -0.47986249, -0.47939602, -0.48018842, -0.48203656]),\n",
       " 'split2_train_score': array([-0.49168732, -0.48830836, -0.48690852, -0.48577019, -0.48664547,\n",
       "        -0.48815149, -0.48931213, -0.48521009, -0.48299922, -0.48253233,\n",
       "        -0.48304499, -0.48494341, -0.4861241 , -0.48199688, -0.48120127,\n",
       "        -0.48105288, -0.48145163, -0.48341777, -0.48326928, -0.48034672,\n",
       "        -0.47907804, -0.47864893, -0.47899218, -0.47993757]),\n",
       " 'split3_train_score': array([-0.49211827, -0.48773604, -0.48647855, -0.48572971, -0.48647904,\n",
       "        -0.48846225, -0.48863525, -0.48582557, -0.48420245, -0.48330473,\n",
       "        -0.48293623, -0.48448826, -0.48646102, -0.48344965, -0.48098539,\n",
       "        -0.48021203, -0.48104387, -0.48356574, -0.48347369, -0.48050971,\n",
       "        -0.47865763, -0.47764662, -0.47929935, -0.48122257]),\n",
       " 'split4_train_score': array([-0.49115515, -0.48810569, -0.48568984, -0.48535727, -0.48721126,\n",
       "        -0.48745275, -0.48771407, -0.48477716, -0.48418959, -0.48268492,\n",
       "        -0.48372477, -0.48570525, -0.48466359, -0.48324247, -0.48089744,\n",
       "        -0.48029041, -0.48064502, -0.48210654, -0.48302949, -0.47957646,\n",
       "        -0.47785658, -0.47802193, -0.47763548, -0.48138788]),\n",
       " 'mean_train_score': array([-0.4920142 , -0.48848833, -0.48661515, -0.48650134, -0.48693903,\n",
       "        -0.48830579, -0.48891641, -0.4858464 , -0.4842321 , -0.48362854,\n",
       "        -0.48400212, -0.48528957, -0.48628897, -0.4836638 , -0.48166199,\n",
       "        -0.48098414, -0.48205625, -0.48368061, -0.48387816, -0.4809165 ,\n",
       "        -0.47927797, -0.47877116, -0.47951708, -0.48147249]),\n",
       " 'std_train_score': array([0.00093261, 0.00069602, 0.00070488, 0.00117628, 0.00043146,\n",
       "        0.00049951, 0.00076585, 0.00108943, 0.00076207, 0.00112792,\n",
       "        0.00110296, 0.00051056, 0.0013717 , 0.00154926, 0.00092856,\n",
       "        0.00101056, 0.00144634, 0.00103179, 0.00127109, 0.0010571 ,\n",
       "        0.00105194, 0.0009068 , 0.00127497, 0.00094317])}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.581985 using {'colsample_bytree': 0.9, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8zdf/wPHXyRIxgoSIxIraJPasaChq1yildmhV1WoVVauqX1W1q6pmjSJ0GK3R2mq3ESOKJEYkkkgEIZH1/v1xr/yCTMlF4jwfj/to7r3nnM/53Cb37XPO57yPEhE0TdM07WmZPe8OaJqmaTmbDiSapmlaluhAommapmWJDiSapmlaluhAommapmWJDiSapmlaluhAommapmWJDiSapmlaluhAommapmWJxfPuwLNgb28vZcqUed7d0DRNy1FOnjx5U0SKplfupQgkZcqU4cSJE8+7G5qmaTmKUupKRsrpoS1N0zQtS0waSJRSbyil/lNKXVJKjU3h/X5KqTCllLfxMTDZezOUUmeVUr5KqXlKKWV8vbtSysf43gxT9l/TNE1Ln8kCiVLKHPgWaA1UAXoopaqkUHS9iNQwPpYY6zYCGgOuQDWgLtBUKWUHfA00F5GqgINSqrmpzkHTNE1LnynnSOoBl0TEH0AptQ7oCJzLQF0BrAErQAGWQAjgAlwQkTBjuT+BLsBf2dt1Tct+cXFxBAYGEhMT87y7ommPsLa2xtnZGUtLy6eqb8pA4gRcS/Y8EKifQrkuSil34AIwUkSuichhpdQeIBhDIFkgIr5KqcJAJaVUGWN7b2IINpr2wgsMDKRAgQKUKVMG40itpj13IkJ4eDiBgYGULVv2qdow5RxJSn8pj++itQUoIyKuGK4uVgIopV4BKgPOGAJSM6WUu4jcAt4H1gMHgMtAfIoHV+pdpdQJpdSJsLCwlIpo2jMVExODnZ2dDiLaC0UphZ2dXZaulE0ZSAKBksmeOwNByQuISLiIPDA+/QGobfy5E3BERKJEJAr4A2hgrLNFROqLSEPgP+BiSgcXkcUiUkdE6hQtmu5t0Jr2TOggor2Isvp7acpAchwor5Qqq5SyAt4GNicvoJRyTPa0A+Br/Pkqhsl1C6WUJdD04XtKqWLG/xYGhgBLTHUCtzZsIOrAAVM1r2maliuYbI5EROKVUkOBHYA5sExEziqlPgdOiMhmYJhSqgOG4akIoJ+x+kagGXAaw3DYdhHZYnxvrlLKzfjz5yJywST9j43l1k/riL18mVLLlmJTs6YpDqNpmpbjmXQdiYj8LiIVRKSciEwzvjbRGEQQkXEiUlVE3ETEQ0TOG19PEJH3RKSyiFQRkVHJ2uxhfK2KiKwzVd+VlRWlfliMRbGiXBv8PjEXTBKvNO2ZiYyMZOHChU9Vd86cOdy/fz/b+rJixQqGDh2abe1l1N69e2nXrl2m62W2v5cvX2bt2rWZPk5GRURE0KJFC8qXL0+LFi24detWiuWuXr1Ky5YtqVy5MlWqVOHy5csm6Y9e2Z4GC3t7Si1dilmePFzzHEhsYODz7pKmPbUXKZDkdmkFkvj4FO8PypTp06fTvHlzLl68SPPmzZk+fXqK5fr06cPo0aPx9fXl2LFjFCtWLMvHTslLkWsrK6ycnSm55Aeu9O7D1QGelFm7Bgt7++fdLS2Hm7LlLOeC7mRrm1VKFGRS+6qpvj927Fj8/PyoUaMGLVq0oFixYmzYsIEHDx7QqVMnpkyZwr179+jWrRuBgYEkJCQwYcIEQkJCCAoKwsPDA3t7e/bs2ZNi+9u3b+fTTz8lISEBe3t7/vrrLyIiIhgwYAD+/v7Y2NiwePFiXF1dH6nn5eXFlClTMDc3x9bWlv3793P58mV69+7NvXv3AFiwYAGNGjVi7969TJo0CQcHB7y9vencuTPVq1dn7ty5REdH8+uvv1KuXDn69euHtbU1Z8+eJSQkhFmzZj1xJXLv3j0+/PBDTp8+TXx8PJMnT6Zjx46pfn7Xrl3jjTfeICAggJ49ezJp0iQmTJiAvb09w4cPB2D8+PE4ODiwdu1afH19qVGjBn379qVw4cJs27aNmJgY7t27x+7du/n666+f+PwBVq9ezbx584iNjaV+/fosXLgQc3PzR/ry22+/sXfvXgD69u3La6+9xldfffVImXPnzhEfH0+LFi0AyJ8/f6rnllU6kGSAdYUKlFz0HVcHeHJ14CBK/7gS84IFn3e3NC1Tpk+fzpkzZ/D29mbnzp1s3LiRY8eOISJ06NCB/fv3ExYWRokSJdi2bRsAt2/fxtbWllmzZrFnzx7sU/lHVFhYGIMGDWL//v2ULVuWiIgIACZNmkTNmjX59ddf2b17N3369MHb2/uRup9//jk7duzAycmJyMhIAIoVK8auXbuwtrbm4sWL9OjRIynx6qlTp/D19aVIkSK4uLgwcOBAjh07xty5c5k/fz5z5swBDFcF+/btw8/PDw8PDy5duvTIcadNm0azZs1YtmwZkZGR1KtXj9dff518+fKleI7Hjh3jzJkz2NjYULduXdq2bYunpyedO3dm+PDhJCYmsm7dOo4dO4arqyszZ85k69atgGFo7PDhw/j4+FCkSBF27tzJxYsXn/j8ixYtyvr16zl06BCWlpYMGTKENWvW0KdPHwYOHMjgwYOpU6cOISEhODoa7lVydHQkNDT0if5euHCBQoUK0blzZwICAnj99deZPn36E0EpO+hAkkE2NWviPG8e14YM4dr7Qyi15AfM8uZ93t3Scqi0rhyehZ07d7Jz505qGm8iiYqK4uLFizRp0oSPP/6YMWPG0K5dO5o0aZKh9o4cOYK7u3vSgrYiRYoAcPDgQTZt2gRAs2bNCA8P5/bt24/Ubdy4Mf369aNbt2507twZMGQBGDp0KN7e3pibm3Mh2Rxl3bp1k75Ey5UrR8uWLQGoXr36I1dL3bp1w8zMjPLly+Pi4sL58+ef+Aw2b97MzJkzAcM6n6tXr1K5cuUUz7FFixbY2dkB0LlzZw4ePMiIESOws7Pj33//JSQkhJo1ayaVSan+w88ltc/fx8eHkydPUrduXQCio6OThqOWLMncDarx8fEcOHCAf//9l1KlStG9e3dWrFiBp6dnptrJCB1IMiF/k1dx+mo61z/6mOsjRuK8YD7qKVMKaNrzJCKMGzeO995774n3Tp48ye+//864ceNo2bIlEydOzFB7Ka1FEHl8DfKTaxYWLVrE0aNH2bZtGzVq1MDb25v58+fj4ODAqVOnSExMxNraOql8njx5kn42MzNLem5mZvbI/MPjx3n8uYiwadMmKlasmO75pdXewIEDWbFiBTdu3GDAgAGp1k9+pZPa5z9//nz69u3L//73vzT74uDgQHBwMI6OjgQHB6c49+Hs7EzNmjVxcXEB4M033+TIkSMmCSR6sj2TCrZpQ/FJE4nat4+g8eORxMTn3SVNy5ACBQpw9+5dAFq1asWyZcuIiooC4Pr164SGhhIUFISNjQ29evXi448/5p9//nmibkoaNmzIvn37CAgIAEga2nJ3d2fNmjWA4Y4pe3t7Cj42LOzn50f9+vX5/PPPsbe359q1a9y+fRtHR0fMzMxYtWoVCQkJmT5fLy8vEhMT8fPzw9/f/4mA0apVK+bPn58U7P79998029u1axcRERFJczGNGzcGoFOnTmzfvp3jx4/TqlUrIP3PK7XPv3nz5mzcuDFpqCoiIoIrV57cEqRDhw6sXLkSgJUrV6Y4t1O3bl1u3brFw8weu3fvpkqVlPLmZp2+InkKhd9+m4TISMLmzMXcthAOn47TK5a1F56dnR2NGzemWrVqtG7dmp49e9KwYUPAMBG7evVqLl26xOjRozEzM8PS0pLvvvsOgHfffZfWrVvj6OiY4mR70aJFWbx4MZ07dyYxMTFpjmPy5Mn0798fV1dXbGxskr78khs9ejQXL15ERGjevDlubm4MGTKELl264OXlhYeHR6rzFmmpWLEiTZs2JSQkhEWLFj1yVQMwYcIERowYgaurKyJCmTJlkuY0UvLqq6/Su3dvLl26RM+ePalTpw4AVlZWeHh4UKhQoaT5B1dXVywsLHBzc6Nfv34ULlz4kbZatmyJr6/vE59/lSpV+OKLL2jZsiWJiYlYWlry7bffUrp06UfmSMaOHUu3bt1YunQppUqVwsvLC4ATJ06waNEilixZgrm5OTNnzqR58+aICLVr12bQoEGZ/hwzQqV06Znb1KlTR7J7h0QRIXT6V0SsXIn9sA8pOmRItrav5T6+vr6pjr9r2atfv360a9eOrl27mvxYiYmJ1KpVCy8vL8qXL2/y45lKSr+fSqmTIlInvbp6aOspKaUoNuYTbDt25Oa8+USYcPGRpmkvpnPnzvHKK6/QvHnzHB1EskoPbaXlwg7IVxScaqX4tjIzw/GLqSTcuUPI1C8wL2iLbbu2z7iTmvZs1a9fnwcPHjzy2qpVq6hevfpz6tGTVqxY8VT1duzYwZgxYx55rWzZsvzyyy8plq9SpQr+/v5PdazcRAeS1MTHwvax8OAueO6EIi4pFlOWljjNnsW1gYMIGjsW84IFyO/u/ow7q2nPztGjR593F0ymVatWSRPmWsbpoa3UWFhBzw2QmACrOkNU6nuamFlb4/zdQvJUKE/gsOHc/yftuz80TdNyEx1I0mJf3hBM7t6AtW/Bg6hUi5oXKECpH37A0sGBa4MHE/OfTvKoadrLQQeS9JSsC28th+BT4NUPEuJSLWphZ0fJpUsxy5uXqwM9ib12LdWymqZpuYUOJBlRsTW0mw2XdsGW4ZDGLdNWzk6UWroEYuO4OsCTeL3Nr6ZpuZwOJBlVux80HQvea2DPtDSL5nnlFUou/p748HCuDhxEwp3szfKqaU/jRUojr/cjyZqM7EeyZ88eatSokfSwtrbm119/NUl/dCDJjNfGQq0+sP9rOL40zaJ53dxwnj+PB/7+XBv8PonR0c+ok5qWshcpkOR2L8J+JB4eHnh7e+Pt7c3u3buxsbFJSnCZ3fTtv5mhFLSdDVGh8PvHkN8BKqf+r5v8jRvj9PUMro8cReCIEZRcsEAnedQM/hgLN05nb5vFq0PrlDc4Ar0fycu2H0lyGzdupHXr1tjY2KRaJktEJNc/ateuLdnqQZTI4mYiU4uJXDmcbvGIdevlXMVKEvjRx5KYkJC9fdFyjHPnzv3/k9/HiCxrk72P38ekefyAgACpWrWqiIjs2LFDBg0aJImJiZKQkCBt27aVffv2ycaNG2XgwIFJdSIjI0VEpHTp0hIWFpZq26GhoeLs7Cz+/v4iIhIeHi4iIkOHDpXJkyeLiMhff/0lbm5uIiKyfPly+eCDD0REpFq1ahIYGCgiIrdu3RIRkXv37kl0dLSIiFy4cEEe/g3v2bNHbG1tJSgoSGJiYqREiRIyceJEERGZM2eODB8+XERE+vbtK61atZKEhAS5cOGCODk5SXR0tOzZs0fatm0rIiLjxo2TVatWJR23fPnyEhUVleL5LV++XIoXLy43b96U+/fvS9WqVeX48eMSEBAgNWvWFBGRhIQEcXFxkZs3bz5ynIf1nZyckj6X1D7/c+fOSbt27SQ2NlZERN5//31ZuXKliIh4enrK8ePHRUTE1tb2kf4VKlQo1f83IiIeHh6yZcuWNMs88vtpBJyQDHzH6iuSp2GVz3Bb8NIWsLa7YcFi0dRTURfu3s2Q5HH2bMxtbXH4bLxO8viyS+PK4VnQ+5Hk/v1IHgoODub06dMmXWhp0kCilHoDmAuYA0tEZPpj7/cDvgauG19aICJLjO/NANpimMfZBQwXEVFK9QA+BQQIAnqJyE1TnkeK8tlB759hSQtY3QU8d0FBx1SL2707iIRbt4hYsQLzwoUpOvSDZ9hZTXuU6P1Icv1+JA9t2LCBTp06YWnCYXWTTbYrpcyBb4HWQBWgh1IqpWT460WkhvHxMIg0AhoDrkA1oC7QVCllgSEweYiIK+ADPPtbPx4qXAZ6bYToW7CmK8TcTrVoUpLHTp24uWABEatWP7t+ahp6P5KXbT+Sh3766Sd69OiR5rlllSmvSOoBl0TEH0AptQ7oCJzLQF0BrAErQAGWQIjxZwXkU0qFAwWBS6k1khUigtcFL4rmLYpHKY/UCzq6QfdVsOYtWPcO9NoEFnlSLKqUwnHq54Ykj9OmYV7IFtv27U3RfU17gt6P5OXajwQMd49du3aNpk2bZvrzy5SMTKQ8zQPoimE46+Hz3hiGrpKX6QcEY7iy2AiUTPbeTCASuA1Me6zdO8Z6+wHz9PryNJPtsQmx0n1Ld6m/pr743fJLv4L3OpFJBUU29BNJZ0I9ISZGLvfqLeeqVpO7e/dmum9azpTSZKZmGn379hUvL69ncqyEhARxc3OTCxcuPJPjmUpWJttNuY4kpdnkxwdMtwBlxDBM9SewEkAp9QpQGXAGnIBmSil3pZQl8D5QEyiBIQCNS/HgSr2rlDqhlDoR9hSryy3NLJnjMYc85nkYtmcYd2LTWVTo1h1enwJnf4ZdE9IsapYnD87fLcS6QgVDkseTJzPdP03Tnj+9H4mByXZIVEo1BCaLSCvj83EAIpLiLJJxTiVCRGyVUqMBaxGZanxvIhAD7AGmi0hz4+vuwFgRaZNWX7KyQ+I/If/gudOTBo4NWNBsAeZm5qkXFjGknj+6CFpOg0ZpT9/Eh4dz5Z1exIeHU3r1KqwzOOmn5Uy5ZYfEnLAfydPK7H4kuUlWdkg05dCWBeAPlMUw13EKqPpYGcdkP3cCjhh/7o7hCsUCw/zIX0B7DFchwUBRY7mpwDfp9SWr60jWn18v1VZUkzkn56RfOCFeZH1vwzCXT/qX1rGBgXKh6WvyX+NX5cGVK1nqp/Zi00Nb2ovshRzaEpF4DHdU7QB8gQ0iclYp9blSqoOx2DCl1Fml1ClgGIY5EzDMl/gBp40B6JSIbBGRIGAKsF8p5QPUAL401Tk81K1iN7pW6MqS00vYfnl72oXNzKHTYijdGH4ZDP770ixu6WRM8hgfz9UBnsQZ79bQNE3LKUw2tPUiycrQ1kNxCXF47vTkfMR5VrVeRcUi6QxDRd+CZa3hdiAM+MOQviKt4j4+XOnXHysnJ0qvXoW5rW2W+qu9eHLL0JaWO2VlaEsnbcwgS3NLZr02iwJWBRi+Zzi3Yp7MtvmIvIUNtwJbF4TVXeHWk/eCP1Lc1ZWSC+YTe/myIcmjTpCnaVoOoQNJJtjntWeux1zC7ocxet9o4hPTyeJp62QIJvHRhtXv9yPSLJ6vUSNKzJxJ9KlTBA4fgcTGZmPvtZfdi5T9V6eRz5qMpJEH+OSTT6hatSqVK1dm2LBhKWYayA46kGRSNftqTGw4kaM3jvLNiW/Sr1CsMrz9E0RehZ/ehri008kXbNWS4pMnce/AAYLGjkMSE7Op59rL7kUKJLndi5BG/u+//+bQoUP4+Phw5swZjh8/zr59ac/ZPi0dSJ5Cx1c60qtyL1b7rmaz3+b0K5RpDF1+gGvHYKMnJKT9i1S4WzeKfjSKO7//TsgXX5jsXxHayyV5GvnRo0fz9ddfU7duXVxdXZk0aRJgSK3etm1b3NzcqFatGuvXr2fevHlJaeQ9PFLP8rB9+3Zq1aqFm5sbzZs3Bwz/cn7zzTdxdXWlQYMG+Pj4PFHPy8uLatWq4ebmhru7O2D4Im7SpAm1atWiVq1a/P3334DhiqJp06Z069aNChUqMHbsWNasWUO9evWoXr06fn5+APTr14/BgwfTpEkTKlSokOKK9Xv37jFgwADq1q1LzZo1+e2339L8/B6mka9YsWJSyvcJEyYwd+7cpDLjx49n3rx5jB07lgMHDlCjRg1mz57NihUreOutt2jfvn1SksmUPn8wpJGvV68eNWrU4L333ksxPcxvv/1G3759AUMa+ZQ2rFJKERMTQ2xsLA8ePCAuLg4HB4c0z/Fp6ey/T2lUnVFcuHWBKX9PwcXWhWr21dKuUKUjtJ4Bf4w27GXSbrZhf5NU2A0cSMKtSCKWLcO8UGGKDvswm89Ae56+OvYV5yPOp18wEyoVqcSYemNSfX/69OmcOXMGb29vdu7cycaNGzl27BgiQocOHdi/fz9hYWGUKFGCbdu2AXD79m1sbW2ZNWsWe/bswd7ePsW2w8LCGDRoEPv376ds2bJJubYmTZpEzZo1+fXXX9m9ezd9+vTB29v7kbqff/45O3bswMnJicjISICkFCvW1tZcvHiRHj168PCGmVOnTuHr60uRIkVwcXFh4MCBHDt2jLlz5zJ//nzmzJkDGILRvn378PPzw8PDg0uXHs2mNG3aNJo1a8ayZcuIjIykXr16vP7666mmYzl27BhnzpzBxsaGunXr0rZtWzw9PencuTPDhw8nMTGRdevWcezYMVxdXZk5c2ZSAFuxYgWHDx/Gx8eHIkWKsHPnTi5evPjE51+0aFHWr1/PoUOHsLS0ZMiQIaxZs4Y+ffo8kiIlJCQkKQOyo6NjUm6u5Bo2bIiHhweOjo6ICEOHDjXZzR46kDwlSzNLZjadydtb32b4nuGsb7ce+7wp/5Elqf8u3LkOh+ZAQSdoOjrVokopio3+mITbkdxcuBDzQoUo0qd3Np+F9rLSaeRzfxr5S5cu4evrS2BgYNLx9+/fn3TVl510IMmCwtaFmdtsLr1/782ovaNY2nIplubppGp+fTLcvQF7voACxaFW6sFBKYXjlCkk3L5NyJdfGpI8duiQankt50jryuFZEJ1GPtenkf/ll19o0KAB+fPnB6B169ZJAT+76TmSLKpUpBJTG0/l39B/+d+xtP/nA4bhrI4LoFwz2DIcLuxMu7iFBU7ffINN/foEjfuUu6lsc6pp6dFp5F+uNPKlSpVi3759xMfHExcXx759+0w2tKUDSTZ4o+wbDKg2AK8LXmz4b0P6FcwtoduPULwaePWFwLSTNprlyYPztwuwrlSJ6yNGcj+Liyu1l1PyNPK7du1KSiNfvXp1unbtyt27dzl9+nTSRO+0adP47LPPgP9PI5/aZHvyNPJubm50794dgMmTJ3PixAlcXV0ZO3Zsqmnkq1evTrVq1XB3d09KI79y5UoaNGjAhQsXspRGvnXr1qmmkY+Li8PV1ZVq1aoxYULayVYfppGvUaMGXbp0eSKNfLdu3VJMIz979uwn2mrZsmWKn3/yNPKurq60aNGC4OBgwHDl83CeaOzYsezatYvy5cuza9cuxo4dCxjSyA8cOBCArl27Uq5cOapXr46bmxtubm60N9G2FXplezZJSEzgg90fcDT4KMtaLaNmsZrpV7obYtiuNzbKsMOiXbk0i8dHRBiSPIaFUXrVj1jrVdI5il7Z/uz069ePdu3a0bVrV5MfKzExkVq1auHl5ZWjMwDrle0vAHMzc75q8hUl8pVg5J6R3Lh3I/1KBRygtzGr6OrOEJV2ni2LIkUotXQJZvnzc3XQu8SmcMmradqzo9PIG+grkmzmF+lHz209cbF1YUXrFeQxT3m3xEcEnoCV7cG+AvTbBnnyp1n8gZ8fV97phVm+fJReuxZLh9T3a9ZeHLnlikSnkc+dsnJFogOJCey+upvhe4bToVwHvmj8RYp3szzhwg74qQe4vAY91xvmUdIQffo0V/v2w9KpBKVXrcK8UKFs6btmOrklkGi5kx7aesE0K9WMIW5D2Oy3mTW+azJWqUIraD8H/P6CzR8aNslKQ97q1XH+dgGxl69w7b3BOsmjpmnPjQ4kJvKe23s0K9mMmSdmciT4SMYq1eoDr30Kp36Cvz5Pt3i+hg0p8c1Mok+fJvDDYTrJo6Zpz4UOJGk4cDGMgJv3nqqumTLjyyZfUqZgGUbvG03g3cCMVWz6CdTuBwdnwdHF6RYv2LIljp9P4d6hQwSNHYs8xf32mqZpWaEDSSpEhDEbffCYuZd28w+weL8f1yPTztz7uHyW+ZjXbB4JksDwPcO5H5eB4SeloM03ULEN/PEJnEs/KWShrl0pNvpj7vz+BzemTtVJHjVNe6Z0IEmFUopNQxrxWdvKmCvFl7+fp/H03XT97m9+PHyZsLsP0m0DoFTBUsxwn8GlyEtM/Htixr7kzS2gy1JwrgubBsKVv9OtYufpid1ATyLXrSds3rwM9U17ubxIaeT1fiRZk9H9SMaMGUO1atWSMjmbig4kaXC0zcvAJi78NvRV9o1+jY9bVuBuTDwTfztL/S//pPfSo2w4fo3b9+PSbOdVp1cZXms4Oy7vYOmZpRk7uJWN4e6tQqUM+5iE+qZbpehHH2HbtQvh3y0iIoUVxNrL7UUKJLndi7AfybZt2/jnn3/w9vbm6NGjfP3119y5cyfLx06JDiQZVNouH0OblWfHSHd2jHBnyGuvcDXiPp9s8qHOtF0MXHmC37yvc+9Byr8k/av2p3WZ1sz7Zx4HAg9k7KA2RQw7LFpYG3ZYvH09zeJKKRwnT6ZAixaE/G86kSnsUaC9vPR+JI/K7fuRnDt3jqZNm2JhYUG+fPlwc3Nj+/btaZ7jUxMRkz2AN4D/gEvA2BTe7weEAd7Gx8Bk780AzgK+wDxAAQWSlfUGbgJz0utH7dq1xRQSExPl1LVbMnXLWak/7U8pPWarVPrsD/lgzUnZfiZYYuLiHyl/P+6+dN3cVRquaSgBkQEZP1DQKZFpTiLfNhC5fyvd4gkPHsjlfv3kXJWqcuevvzJ5VpqpnDt3Lunn4GnT5HKv3tn6CJ42Lc3jBwQESNWqVUVEZMeOHTJo0CBJTEyUhIQEadu2rezbt082btwoAwcOTKoTGRkpIiKlS5eWsLCwVNsODQ0VZ2dn8ff3FxGR8PBwEREZOnSoTJ48WURE/vrrL3FzcxMRkeXLl8sHH3wgIiLVqlWTwMBAERG5dcvw+33v3j2Jjo4WEZELFy7Iw7/hPXv2iK2trQQFBUlMTIyUKFFCJk6cKCIic+bMkeHDh4uISN++faVVq1aSkJAgFy5cECcnJ4mOjpY9e/ZI27ZtRURk3LhxsmrVqqTjli9fXqKiolI8v+XLl0vx4sXl5s2bcv/+falataocP35cAgICpGbNmiIikpCQIC4uLnLz5s1HjvOwvpOTU9Lnktrnf+7cOWnXrp3ExsaKiMj7778vK1c3qKt6AAAgAElEQVSuFBERT09POX78uIiI2NraPtK/QoUKPdHnHTt2SKNGjeTevXsSFhYmZcuWlZkzZ6b2v/CR38+HgBOSge96k6WRV0qZA98CLYBA4LhSarOInHus6HoRGfpY3UZAY8DV+NJBoKmI7AVqJCt3EvjZNGeQPqUUrs6FcHUuxKdtKnP8cgRbfIL4/fQNtvoEU8DaglZVi9PerQSNy9mR1yIvczzm0GNrD4bvGc6aNmvIb5X2KnYAHF3h7dWwuiuse8dwlWJpnWpxMysrnOcv4Gq/flwfMZKSS34gX7162XjmWk6n9yPJ/fuRtGzZkuPHj9OoUSOKFi1Kw4YNsbAwzVe+KfcjqQdcEhF/AKXUOqAj8HggSYkA1oAVhisRSyAkeQGlVHmgGJDBcSLTMjNT1Hexo76LHZPbV+WQXzhbTgWx48wNNp4MpEg+K9pUL0571xLMcP+awX++x7iD45jrMRczlYERRpfX4M3v4OeB8Mt70HU5mKVezzx/Pkr+sJgr7/QicMgHlP5xJdZVqmTb+WpZU/zTT5/r8UXvR5Lr9yMBw1Db+PHjAejZs6fJ8oGZco7ECbiW7Hmg8bXHdVFK+SilNiqlSgKIyGFgDxBsfOwQkcdnm3tguJpJ8TYopdS7SqkTSqkTYWFhWT2XTLEwN6NphaLMfMuN45+9zve9a9OonB0bTwbSffERRqyIwjVvb/Ze28t33t9lvGHXt6DFVDj3K+z4NN3V7xaFCxuSPBYoYEjyePly1k5My9H0fiQv134kCQkJhIeHA+Dj44OPj0/S1Vt2M+UVSUoJph7/5tsC/CQiD5RSg4GVQDOl1CtAZcDZWG6XUspdRPYnq/s2kOr2giKyGFgMhlxbT3kOWWZtaU6rqsVpVbU49x7E86dvCFtOBbPPuxLmxWqzyGcR568W4MMGnajgUCD9Bht9CHeD4chCKOgIjYenWdzS0ZFSS5dy5Z13uDrAk9I/rcXSwSGbzk7LSZLvR9K6deuk/TAA8ufPz+rVq7l06RKjR4/GzMwMS0tLvvvO8A+dh/uRODo6PjJ89FDy/UgSExOT9lyfPHky/fv3x9XVFRsbm1T3I7l48SIiQvPmzZP2I+nSpQteXl54eHhkaT+SkJCQVPcjGTFiBK6urogIZcqUSXFS/qGH+5FcunSJnj17PrEfSaFChVLcj6Rfv34ULlz4kbZatmyJr6/vE59/8v1IEhMTsbS05Ntvv6V06dKP7Nk+duxYunXrxtKlSylVqhReXl6AYT+SRYsWsWTJEuLi4pKGJgsWLMjq1atNNrRlsqSNSqmGwGQRaWV8Pg5ARFK8ZjPOqUSIiK1SajRgLSJTje9NBGJEZIbxuRvgJSIVMtKXZ520MSNu349j2+mrzPUdyb3E69y//AHlC79CezdH2ruVoLRdGn84iYmwaQCc/QU6/wCu3dI9XvSZs1zt0weLEo6UXrUKi8d+sTXT00kbnx29H0nmvahJG48D5ZVSZZVSVhiuIB5Zpq2Uckz2tAOGO7QArgJNlVIWSilLoGmy98AwrPWTyXr+DNjaWNKzfjk2v/UDdjYFcKq0DhvrB8zceYGmX++l44KDLDngz43bMU9WNjODTt9DmSbw6xDwS3/73bzVquK8cCFxV69xbfBgEu89XeoXTdP+n96PxMCkaeSVUm2AOYA5sExEpimlPsdwS9lmpdT/MASQeCACeF9EzhuvThYC7hiGw7aLyKhk7foDbUTkPBnwIl6RJOcd6k3/Hf2pV7we4+t8w/bTIWw+FcSZ63dQCuqWKUJ7txK0qVYcu/zJ9jeJjoTlbSDyCvT/HRzd0j3W3T//JHDYcPI1aIDzou8ws7Iy4ZlpyeWWKxK9H0nupPcjSceLHkgANl7YyJTDU+hfrT+jahtipn9YFFt9gtl8KohLoVGYmykav2JPB7cStKzqQEFrS7gTBEtaQGIceO6EwmXSPVbkpp8JHj+eAm+8gdM3M1HGcV3NtHJLINFyp6wEElNOtmuZ0LVCV85HnGf5meVUKlyJNi5tcCman2HNy/Nhs1c4f+MuW04FsflUEB97ncLqZzNeq1iUDjVK8Hr3DVivamNY/T5gJ+RL+T72hwp16UzC7duEzpjBjYIFKT5lcsY239KyLLXbZDXtecrqBYW+InmBxCXEMXDnQM6Fn+PH1j9S2e7Jf72KCN7XItl8KohtPsGE3n2AjZU575YO4cPrH6Mc3TDru9mQqysdod/MIvyHH7B77z2KjRxhilPSkgkICKBAgQLY2dnpYKK9MESE8PBw7t69m7Sg9CE9tJVMTgkkADejb/L21rcxU2asa7eOItZFUi2bkCgcC4hg86kg/jgTTP2YQ3xnNZfzBRoR0W4ZDV4phoV56vdTiAg3Jk4i0suLYmPGYNe/nwnOSHsoLi6OwMBAYmJSuIFC054ja2trnJ2dsbR8dItvHUiSyUmBBODszbP03d4X16KufN/ieyzN0t6/HSAuIZGDl24SvvtbuobMYW28B7PyDKGtawnau5WgVqnCmJmlsPI4IYHrI0dxd+dOHL/8kkKdO5nilDRNy4F0IEkmpwUSgC1+W/j04Kf0rNSTcfXHZapu/M4pWPw9i61F+vJRaGsexCfiVCgv7VwNa1Sqlij4yNBKYmwsgYMHc+/oMZznzaWAMXOrpmkvtxdhHYmWBe3Ltad3ld6sPb+WXy5m7tZDixYTwa0n7SJWcqr9DWZ3d6Ni8QIsPRhAu/kHaf7NPmbtusClUEMKB0OSx/lYV61quDr56y9TnJKmabmUviJ5gcUnxjP4z8H8E/IPK95YgWtR1/QrPZQQZ9gQy28PvL0WKr7BrXuxbD97gy2ngjjsH44IVHYsaFhN71oCR/WAqwM8eeDrS8H27XH4dJxeAa9pLzE9tJVMTg0kAJExkby97W3iEuJY124dRW2KZrzygyhY0RbC/oO+W6Bk3aS3Qu/EsO10MFtOBfHP1UgAapYqRIcqRWnx7x/cXboE84IFKf7ZeAq0bq3vMtK0l5AOJMnk5EAC8F/Ef/T+ozcVCldgWatlWJlnYjV6VCgsbQkxt8FzF9i/8kSRaxH32epjCCrngu9Qr2wRlrsXJnTCBGLOnCF/s2YUnzQJS4eUU1VrmpY76TmSXKRikYpMbTyVU2Gn+PLol5lbPJS/mGEjLGUGqzvB3ZAnipQsYsP7r5Xj9+FNmPmWG8cCIph3WSiz7ieKffIJ9w4dwr9dOyI3bszywiVN03IfHUhyiFZlWjGo+iA2XdzEhv82ZK6yXTl4ZwPcuwlrukLMnVSLdq3tTM/6pfh+nz+7/ruJ3YD+uGz+DetKlQj+bAJX+w8g9tq1VOtrmvby0YEkB/mgxge4O7sz/dh0ToaczFxlp9rQ7UcIOQsbekN8bKpFJ7arQnUnWz7yOsWV8HtYlS5NqZUrKD55MjGnT+PfoSMRK1ciT7HZkKZpuY8OJDmIuZk505tMx7mAM6P2juLGvRuZa6B8C+gwH/z3wuahhn1NUmBtac7Cd2phphSDV/9DTFwCysyMwm93x2XbVvLVq0fI/6Zzpec7PLh0KesnpmlajqYDSQ5TwKoAc5vN5UHCA4btHkZMfCbTbdR8B5p9Bj7r4a/JqRYrWcSG2d3d8A2+w8TfziS9blm8OM6LvqPE118Te+UKAZ06E7ZwIRKb+hWOpmm5mw4kOZCLrQvTm0znfMR5phyekvkJ8CYfQ50BcGguHFmUarFmlRz4wKMcG04EsuH4/8+LKKWwbd8Ol21bKdCiBTfnzSfgrW5Enz6TaluapuVeOpDkUK+VfI0PanzAVv+t/Hjux8xVVgrazIRK7WD7WMOWvakY1aIijcrZMeG3M5wNuv3IexZ2djjN+gbnhd+ScOsWl7t3J3TmTBJ1UkJNe6noQJKDDXIdxOulXmfWyVkcDjqcucpm5tBlCZSsBz+/C5cPpljM3Ewxr0dNCtlYMmTNP9yOjnuiTIFmzXDZuoVCXToTvmQpAR3f5P7x409zSpqm5UA6kORgZsqMaa9Ow8XWhdH7R3PtbiZvy7XMCz3WGXZV/KknhJxLsZh9/jx827MW129FM9rrVIpDaeYFC+I4dSqlli9DEhK40rsPwVOmkBAV9RRnpmlaTqIDSQ5nY2nDPI95iAjD9wznftz9TDZQxLBg0crGsMPi7cAUi9UpU4SxrSux81wIi/f7p9pcvoYNcdn8G0X69iVy3Xr823cgav/+zPVJ07QcJd1AopQqp5TKY/z5NaXUMKVUoYw0rpR6Qyn1n1LqklJqbArv91NKhSmlvI2Pgcnem6GUOquU8lVKzVPGZE9KKSul1GKl1AWl1HmlVJeMn27uVLJgSb52/xq/SD8+O/RZ5iffC5WCdzZCbJQhmETfSrGY56tlaV2tODN2/MdR//BUmzOzscFh3FjK/LQWs3w2XHv3PYLGjCH+VsrtapqWs2XkimQTkKCUegVYCpQF1qZXSSllDnwLtAaqAD2UUlVSKLpeRGoYH0uMdRsBjQFXoBpQF2hqLD8eCBWRCsZ292XgHHK9Rk6NGFlrJLuu7GLJ6SWZb6B4Nei+GsL9YGkruHnxiSJKKWZ0daVUERuG/vQvoXfTnlTPW6MGZX/+Gfsh73N72+/4t23HnT/+0GlWNC2XyUggSRSReKATMEdERgKOGahXD7gkIv4iEgusAzpmsF8CWANWQB7AEniYJGoA8D8AEUkUkZsZbDPX61u1L23KtmH+v/PZH/gUw0kuTQ3DXPdvwg/N4Py2J4oUsLbku161uBsTx4dr/yU+IeVFjQ+ZWVlRdNgwym7aiKWjI9dHjiLwww+JCw3NfP80TXshZSSQxCmlegB9ga3G19Lf+xWcgOSzv4HG1x7XRSnlo5TaqJQqCSAih4E9QLDxsUNEfJMNqU1VSv2jlPJSSjlkoC8vBaUUkxtNplKRSozZP4aA2wGZb8SlKby7z5Cfa11P2P0FJD6aCqVS8YJMe7M6RwMimLnzQoaata5YkTLr11Fs9MfcO3AQ/7btiNy0SV+daFoukJFA0h9oCEwTkQClVFlgdQbqpbSBxePfGluAMiLiCvwJrAQwDqNVBpwxBJ9mSil3wML42iERqQUcBmameHCl3lVKnVBKnQgLC8tAd3OHvBZ5mesxFytzK4btHsbd2LuZb6RQSei/HWr0gv1fw9ruT8ybdKntTI96pVi0z49d557MKJwSZWGBnacnZX/9BeuKFQke/xnXPD2JDUx5gl/TtJwh3UAiIudEZJiI/KSUKgwUEJHpGWg7ECiZ7LkzEPRY2+Ei8sD49AegtvHnTsAREYkSkSjgD6ABEA7cBx6uoPMCaqXS78UiUkdE6hQtmonNoHIBx/yOfNP0GwLvBjLuwDgSJe3hpxRZWkPHBdB2liE31+LX4MajK9cnta9CNaeCjNrgzdXwjN8tlqdsWUr9uJLikycRfcoH//YdiPhxlU4CqWk5VEbu2tqrlCqolCoCnAKWK6VmZaDt40B5pVRZpZQV8Daw+bG2k8+1dAB8jT9fBZoqpSyUUpYYJtp9xTAOsgV4zViuOZDy4oeXXJ3idfik3ifsC9zHt97fPl0jSkFdT+j/O8Q/gCWvw+mNSW9bW5rz3Tu1UcD7a04SE5fxQGBIAvk2Llu3YFO3DiFffsmVd3rxwM/v6fqqadpzk5GhLVsRuQN0BpaLSG3g9fQqGSfohwI7MASIDSJyVin1uVKqg7HYMOMtvqeAYUA/4+sbAT/gNIbgdUpEthjfGwNMVkr5AL2BjzJwDi+ltyu+TefynVnss5hdV3Y9fUMl6xnmTUrUgE2esP1Tw57wPEzuWIOzQXeYvPlsppu2dHSk5PffU2LGV8QGBBDwZiduLlqExD25gl7TtBdTulvtKqVOAy0xzF+MF5HjSikf47xGjpDTt9rNitiEWPrv6M/FWxdZ3WY1FQpXePrG4mNh52dw7Hso/Sq8tQLyG4YNZ2w/z8K9fnzd1ZW36pRMu53Umg8P58YXX3D3j+3kqVQJx2lfkLdq1afvr6ZpWZKdW+1+juGqws8YRFyAJxcZaC8kK3MrZr82m/yW+Rm+ezi3H9xOv1JqLKygzQzo9D1cPwGLm0KgYYOtUS0q0NDFjs9+PcO5oNR3YEyzeTs7nGfPxnnBfBLCw7ncrTuh33yjk0Bq2gsuI5PtXiLiKiLvG5/7i8hLv5o8JylmU4zZHrMJuR/C6H2jiU+Mz1qDbm+D505D4sflb8DJlViYmzGvR01s81oyZM1J7sQ8/dBUgddfx2XrFmw7vUn4D0sIeLMT91/SK0pNywkyMtnurJT6RSkVqpQKUUptUko5P4vOadnHragbExpM4HDwYeacnJP1Bh3dDPMmZV6FLcNgy3CK5oVv36nFtVvRfOLlk6U1Iua2tpT44gtKLVuKxMVxpVdvbnw+lYSoe1nvu6Zp2SojQ1vLMdxtVQLDmo4txte0HKZT+U68XfFtVp5byVb/relXSI9NEUOOrldHwskVsLwNdQtHM/aNSmw/e4MlB55iQeRj8jVqhMuWzRTu05tbP/2Ef4f2RB04kPW+a5qWbTISSIqKyHIRiTc+VgAv18KMXOSTep9Q26E2k/+ezNnwzN9l9QQzc3h9MnT7EcLOw+KmDCx5nTeqFmf69vMcC4jI+iFsbCj+6aeUXrsGs7w2XBv0LkFjxpIQGZnltjVNy7qMBJKbSqleSilz46MXhoWBWg5kaWbJN02/obB1YUbsGUF4dDb9r6zSEQb+Bda2qB87MqfMYUoWsmbo2n/STe6YUTY1a1L2l5+xe38wt7dtw69tO+5s35EtbWua9vQyEkgGAN2AGxjyXnXFkDZFy6Hs8tox12Mut2Ju8dG+j4hLzKY1G8UqwaDdUOENrP8az28lfiQ2JorhP3mnm9wxo8ysrCg2fDhlN3ph6eDA9REjCPxwmE4CqWnPUUbu2roqIh1EpKiIFBORNzEsTtRysCp2VZjSaAonQ04y49iM7GvY2taQjt7jM2wv/crewl8SGHCOWbsyltwxw4epVIkyG9ZT9KNRRO3bh3+79kRu+lkngdS05+Bpd0gcla290J6Lti5t6Ve1H+v+W8fPF3/OvobNzKDpaHjHi0KxN9hhM5Gz+3/mzwwmd8woZWGB/aBBlP3tV/JUKE/w+PFc8xxIbOD1bD2Opmlpe9pAklJmXy0HGlFrBI1KNOKLI1/gHeqdvY2XbwHv7sXarhTLrWZwbsNErt7M/j3c85QtS+kff8Rh4gSivb3x79CBiFWrkcTsGU7TNC1tTxtI9PhBLmFuZs4M9xk42Dgwau8oQu9n81xDERfMBu4iusKbDGMd17/vQkxU9m+5q8zMKNKzJy5bNmNTuzYh06YZkkD6p76/vKZp2SPVQKKUuquUupPC4y6GNSVaLmGbx5Z5zeYRFRfFyD0jiU2Izd4DWOUjX4/l/FfjU+rGHuPOfHcI+y97j2Fk6eREycXfU+Kr6cT6+xPQ8U1uLvpeJ4HUNBNKN2ljbvAyJ23MjD+v/MnIvSOxz2tPE6cmuDu707BEQ/JZ5su2Y6xdv4aW58ZiaxGPZdfFULl9trX9uPibN7kx9Qvu7thBnsqVKTHtC6yrVDHZ8TQtt8lo0kYdSLRH7L22ly1+W/g76G+i4qKwMLOgtkNt3J3ccXd2p4xtmSy1H5+QyLDFW3nvxmTclB80+Qg8xhsWNprInZ07uTF1KgkRt7Dz9MT+gyGY5cljsuNpWm6hA0kyOpBkXlxiHN6h3hwIPMD+wP343TZsOFWqQCmaODfB3cmdOsXrYGVulem2Q+/G0GnuHsaxjHbxO6Fcc+iyxJByxUQSbt8m5KsZ3P75Z6zKlsVx2hfY1Epxc01N04x0IElGB5Ksux51nf2B+zkQeIBjN47xIOEBeS3y0sCxAe7O7jRxaoJDPocMt3csIIIePxxhitNx3olYgCrgaFh/4mjabW6iDh3ixoSJxAUHU7hnT4qOHIl5/uwbutO03EQHkmR0IMle0fHRHL9xnP2B+9kfuJ/ge8EAVCxcEXdnwxBYdfvqmKczXLV4vx9f/n6eua/G0/G/sRB9CzrMA9duJu1/4r17hM6Zy63Vq7FwLI7jlM/J3+RVkx5T03KibAskxru0Hi90GzgBfCQiL/z9lTqQmI6I4Bfpx/7rhqDiHepNgiRgm8eWxiUa4+7szqtOr2KbxzbFuoNXn+RP31A29i5HzSMj4cohqP8+tJwK5pYm7fv9f/4h+LMJxPr7U7h3bxw+GY2yNO0xNS0nyc5AMgUIAtZiWIj4NlAc+A94X0Rey3JvTUwHkmfn9oPbHA46zP7A/Ry8fpBbD25hpsxwK+qWNARWoXAFlDKsab0TE0eH+QeJjktg2wcNsD88DY4shNKNjVv5FjNpfxMfPCBs1iwiVv5I3tq1cZo9C8tipj2mpuUU2RlIjopI/cdeOyIiDZRSp0TELYt9NTkdSJ6PhMQEzoafTRoC843wBcDBxiFpwr6+Y30uh8XTaeEhapcuzCrP+pif2QibP4S8haDbKihZ1+R9vb1tG8GfTcAsfz6c587VE/GaRvYGksPAbGCj8aWuwChjIPEWkRpp1H0DmAuYA0tEZPpj7/cDvgYeJkdaICJLjO/NANpiWDS5CxguIqKU2gs4AtHGOi1FJM3l2DqQvBhC74dy8PpBDgQe4O+gv7kffx8rMyvqFq9L/sRqbDpYkCGv1mN0q0pw4zSsewfuBBn2ia/dH5RpM/PEXLjA9Q+HEXv9Og5jxlC41ztJV06a9jLKzkDigiEYNDS+dBgYieHLv7aIHEylnjlwAWgBBALHgR4ici5ZmX5AHREZ+ljdRhgCjLvxpYPAOBHZawwkH4tIhiODDiQvnriEOE6Gnky6E+zyncsAJDwoSvPSTenl+ga1CpTB8tchcOlPqNkb2swES2uT9ivh7l2CxowlavduCrZvj+OUyZjZ2Jj0mJr2ospoILFIr4BxMj215ccpBhGjesClh5PxSql1QEfgXBp1kg4LWANWGOZlLIHsTR2rPVeW5pY0cGxAA8cGfFL3E67eucruq/v49sgW9gb/wt4bG8lnmY+GTg1wL9iZJqfWYB9yFrqvAltnk/XLvEABnBfMJ3zxYsLmzuPBf//hPH8eVqVLm+yYmpbTpZu0USnlrJT6RSkVqpQKUUptUkpl5C/ZCbiW7Hmg8bXHdVFK+SilNiqlSgKIyGFgD4aNtIKBHSLim6zOcqWUt1JqgtJjD7lCqYKl6FetN5s6rYArn1Ps/mBalGqFz83TTLx1Ao9SznRXN/h29ev4nPqRRDFdZl9lZob94MGUXLyY+JAQArq+xd09e0x2PE3L6TKS/Xc5sBlDokYnYIvxtfSk9AX/+DjaFqCMiLgCfwIrAZRSrwCVAWfjMZsppR4Oc70jItWBJsZH7xQPrtS7SqkTSqkTYWFhGeiu9iIoZWfDrLfq43elDAlhXfmz659sbL+RYTWHYeVQlcX5LHnH+2s81tRn/IFP2X55O3di75ikL/mbvEqZTZuwLOlM4PtDCJs3X6em17QUZGSO5IkJ9fQm2Y1lGgKTRaSV8fk4ABH5XyrlzYEIEbFVSo0GrEVkqvG9iUCMiMx4rE4/UphjeZyeI8l5/veHL9/v82dWNzc61/r/C+DIO4Ec3PoeB275cih/QW6TgLkyp2axmkl3gpUrVC5bJ8kTY2K4MeVzbv/yC/ncm+A0YwbmhQplW/ua9qLK6BxJRq5IbiqleimlzI2PXkB4BuodB8orpcoqpawwrD/Z/FgnHZM97QA8HL66CjRVSlkopSyBpoCv8bm9sa4l0A44k4G+aDnM6JYVqV+2CJ/+cprzN/7/iqNQQWfa9djKV65D2Xv5Cj9G56V/uU7cib3D7JOz6bS5E29seoMvjnzB/sD9xMTHZLkvZtbWOH45jeKTJ3Hv8BECur5FjK9v+hU17SWRkSuSUsACDHdtCfA3MExErqbbuFJtgDkYbv9dJiLTlFKfAydEZLNS6n8YAkg8EIFhgeN549XJQgx3bQmwXURGKaXyAfsxTL6bYxgOGyUiCWn1Q1+R5Eyhd2NoO+8g+fNYsHloYwpYP7bq/NJfsMkTJBG6LOVGieocuG5IMnk0+CjR8dHkMc9DveL1klK3lMifta10or29CRw+goTISIpPmUyhN9/MUnua9iIzaa4tpdQIEZnzVD17DnQgybmO+ofTc8lRWlV14NuetZ4csrp1Gdb3ghtnwONTaPIxmJnxIOEBJ26cSAos1+4a7vt4pdArNHFuQhOnJtQoVgNLs8ynRIm/eZProz7i/rFjFO7ZA4exY1FWmc+CrGkvOlMHkqsiUuqpevYc6ECSs32/z4///XGeCe2q4Plq2ScLxN6HrSPAZz1UbAOdFoH1/+f2EhEu37mctGblZMhJ4iWeApYFaOTUCHdndzxKelDAqkCG+yTx8YTOmk3EsmXkrVEDp7lzsHTIePZjTcsJTB1IrolIyafq2XOgA0nOJiK8t+oku8+Hsu7dBtQpk8K+JSJwbDHs+BQKl4Hua6BYpRTbi4qN4kjwEUNguX6Am9E3sTa3pmWZlnR6pRO1HWpneLL+zvbtBH06HjMbG5xnz8KmrunTuWjas6KvSJLRgSTnux0dR4cFB4mJS2DbsCbY509lh8Mrf8OGvhB7D95cCFXTnsNIlETO3DzDr5d+5Y+AP4iKi6J0wdK8+cqbdCzXkaI2RdPt24NLlwgc+iGx167h8MloCvfpo1OraLlClgNJKunjwbA+JK+IpLsq/kWhA0nucDboNp0X/k2dMoX5cUB9zM1S+bK+EwQb+kDgcWg8AppPzNBWvtHx0ey6sotNFzbxz/+1d9/xUVV5H8c/JzNpJISETkINIC30JkWUWEBEBBQRrOtaX4I+6u66ru6uq66u7uOu3V11eVbBSkdUbAkiofdAQEInIRBISEidTPk9f9wBxpBKCgn5vV8vXsnMvXfmHELmy4TbqdUAACAASURBVLn33N9J34TN2Lis7WVM7jKZy9peht2v9H/y7txc0p58kpzvvids3DjaPPcsfiG6YJaq33RhKx8aJBePz9cf5nfztzEztguPX9Ot9B1dDlj2e9gwC6KvgBtnQUizCr/PgewDLNyzkMV7FpNRmEGL4BZM6DyBSV0n0SGs5HIpIkLGe+9z/NVXCewcTds33iCgY8dK9U+pukSDxIcGycXld/O28vmGFP7vrsGM7l7O2iGbZsOXj0NoK6tOV2SZ99Gew+lx8lPKTyxMXsiK1BV4xMOgVoOY3HUyV3W4imB78DnH5K1aRepjjyMuF5Evv0Tj2NhKvadSdYUGiQ8NkotLodPN5LdXkZpVwNKZI2nXtJzqvKkb4bM7IP8EjH8V+k07r/dNz09nyd4lLEhewOGcw4T6hzKu0zgmXzKZnk17/uK6iDM1lZRH/ofC7dtp9sD9tJg5E2Mr//SaUnWJBokPDZKLz8GMPMa/sZJOzUOY+8AwAu3lfEjnnYC5d8GBn2DwvTDmBbCf370fIsKGYxtYkLyA7w5+h8PtoFtENyZ1ncT46PFnlhX2OBwce/55subOI2TkSCL//jL2iIjzek+lLgQNEh8aJBenb3Yc5f7ZG7nt0vY8P7F3+Qe4XfDDM7DqDWg/zFrKt3HrKrXhVNEpvt73NfOT57MzcycBfgFc2eFKJnedzJDWQ/Azfpz8/HOOPfc89pYtiXr9NYJ79arSeypVWzRIfGiQXLxe/Gon/16xj1en9mNi/5JWKSjB9vmweAYEhsHNH0L7oeUfUwG7MnexIHkBS/ctJacoh6jQKCZ1mcQNXW6gyd50q7RKRgat//xnwm+cXC3vqVRN0iDxoUFy8XK5PUx/fy2JKdksnjGCS1pV8O70Y0nw6XTIToFr/waDfl1tS/kWugr54dAPLExeyNqja/EzfgyPHM6UFtcQ/Y9FFKxdR/jUqbR66g/4aWkVVYdpkPjQILm4pZ8qZNzrKwkLtrNkxkhCAyt4i1NBFiy4D5K/gZAW0HYwtB0EbYdAZH8IDK1y2w7nHGZh8kIW711Men46zf0jeHxjazp/mUhQ3z60fe01/FtX7fSaUjVFg8SHBsnFb82+DKa/t4ZrY9rw5vT+Fb+z3OOxanTtX2HdwJiRbD1v/KBlL2+wDIZ2Q6BpZ/CryMoL53J73CQcSWBB8gJ+PPwjA3cWMeMrsAUFE/WPV2g64vLzel2lapIGiQ8NkobhneV7eWnZLv58fU9+NaKE4o4VkZ8JqZusUElZBykbwZFtbQsKPxssbQdB1CAIrvwCVycKTrB071JWJHzCtA8P0SYTEqf0peeM39OnRV8tr6LqDA0SHxokDYOIcO+HG1n+czqf3T+MgR2qYaqtx2ONUlLWW38Or4f0JM5UD2rezeeU2GBo2aNC5VhOt3fLwdWkP/Un2m9MZXV3w7JpXbgu5kau73w9TYNKKE6pVC3SIPGhQdJwZBc4uf6NlTjdHpbOHEmz0oo7VoUjx2fUssEaueR7Fw0NCIWoAd5wGWyNWkLLLvwoIqS9+y+yXnuDEy0Cef4GJ8db+DO63Wgmd53MsDbDsFUwnJSqThokPjRIGpbtqdlMfmcVQzo25YO7h5Re3LG6iMDJ/d5Q8Y5cjiaCx2Vtj+hoXcA/PXJp3Rts5y6olbdmDamPPobbUcj6e4fxXsQ2shxZtA5pzcQuE5nYZSJRoRWc4qxUNdAg8aFB0vB8tv4QT8xP5OHYLjxWVnHHmlKUD2lbzwZLynrISbO22YOgTT9oN/jsyCXMWgLYmZZmlVbZto3we+5m26QYFu5bzKojqwAY2mYoN3a9kdj2sQTYdOqwqlkaJD40SBqm387dytyNKfzfrwYzuls5xR1rQ3aq9wK+d+RyZAu4Hda2sKgzU489Lftx7INlZM2dT8jwYUS+8grH/QtZtGcRi/Ys4kjeEZoENmF89HgmdZlEt6YXIChVg6BB4kODpGEqKHIz6e0Ejp4qZOnMkbSNKKe4Y21zFcGxROsC/ulRS9ZBa5ufP1knOnM0LhdbeGPavvQXgodfgwdhTdoaFiQvIO5QHE6Pk5hmMUzqOolxncYRGlD1e1+UOq1OBIkxZizwGmAD3heRvxXbfhfwdyDV+9SbIvK+d9vLwHWAH/Ad8Ij4NNYYswSIFpGY8tqhQdJwHTiRx/VvrCS6RQifV6S444WWm372An7KBgq2bSF1eTCuQhuthruJuHLQmZFLVrNolqbEMz95Pnuy9pxZLnhy18kMaDlApxGrKrvgQWKMsQG7gauBFGA9ME1Eknz2uQsYJCIzih07HCtgRnmfWgk8KSLLvdsnAzcBfTRIVHmWbT/KA3M2cvulHXhuYrn/XOoWtwtX8lqOPP0cedsPEt7TTqteh6wZxt6bJiVqINubtWWBI42vj64iz5lHx7COTOo6iQmdJ9A8uPmF7oWqpyoaJOd3m27FDAH2iMg+ESkCPgVuqOCxAgQBAUAg4A8cAzDGhAKPAc9Xe4vVRWlsTGvuGxXN7DUHWbwltfwD6hKbHXv3EbT77Eua3X8/WUkuDu6IxXn1ezDqdxDaErNjIb2/e54/r/gPcYfTeM60oqkjj39u/CdXzb2Kh+Me5sfDP+I6PYtMqWpWk+uuRwGHfR6nACWVWb3RGDMKa/TyqIgcFpHVxph4IA1rjfg3RWSnd//ngFeA/LLe3BhzH3AfQPv27avUEVX//W5MN7YcyuL38xPp0Sas4sUd6whjs9Hy0f8huHcMR574Pft/8ypR/3iFkNuf/MVNk41S1jMxZQMT9yexz+7HotBQFrvjiT8cTwt7CDe0u4pJfX5N+/DzvPNfqRLU5KmtKcAYEbnH+/h2YIiIzPTZpxmQKyIOY8wDwM0iEmuM6YJ1bWWqd9fvgCeAU8BzInK9MaYjsFRPbamKOu/ijnWMY/9+UmbOpGjfflo8+j80u+eec6+H+Nw06Ty8nhUnNrEwQPgpOAiPMQwmiHvajWHY4IcxjevAjDZVJ9WFayTDgGdEZIz38ZMAIvJiKfvbgEwRaWKM+S0QJCLPebf9CSgEcoA/AkVYo6mWwCoRuaKstmiQqNNW783g1vfXMK53G96YVonijnWMJy+PI08/Tc7Xy2h89VW0efFFbKFlzNgSgZMHOLbvB5bs/4rPc/dw1E8YXODg4Uad6dfnDuh+HQSF1V4nVJ1XF66RrAe6GmM6GWMCgFuAJb47GGPa+DycAJw+fXUIuNwYYzfG+AOXAztF5B0RiRSRjsBIYHd5IaKUr2Gdm/GbMd1Yui2ND1YduNDNOW9+ISFE/eMftHziCXLi4jkw5WYce/eWfoAx0LQTrQbdw71TFvDlHRt4ssev2BcSxu2SwozVf+Tn13rA53fCzqXgctReZ1S9V9PTf8cBr2JN/50lIn81xjwLbBCRJcaYF7ECxAVkAg+KyC7v6ORtrFlbAiwTkceKvXZH9NSWOg8ej3Df7A38uPs4n90/jAHt6/c66nlr15H62GNIQQFtXniBsLFjEBEcLg8Op4dCl5tCp5tCp4cCp/W9AQZ2iKDIU8jHOz9mVuJ75LjyGVvo4qHj6XS0h0CPCdDnZugwosKFKNXF5YKf2qpLNEhUcdn5Tsa/+RMut9RccUcfpz/YT3+gFzrd3g/4088V/95NYbH9HS43BUXex65fHhOcncHd379Ll+MHWNhtNP/pPha3KfvDPyYqjBcm9aZP23BOFZ3igx0fMDtpNkUuBzf4N+eBw7tpU3AKGreBmBuh901WaZd6ejpQVZ4GiQ8NElWS08Udh3ZqynM3xFBY7IPaUfzD3eWxtrvc1v/0fQPA+8Fe4PR4j/vl8w6Xh/P9VQuw+RHo70eQv40gfz+C/W3W93abz/M2GuFm+DezuWTNt2R07U3SPb/D1qwZQf5+BNltZ44P8reRll3Iy8t2cSLXwR3DOvL4NZfQOMifjIIM3k98n89+/gyAm5sP4J6T2TTfEw8eJzTrAr2nWH+ada7Gn4aqizRIfGiQqNJ8su4QTy5IrNQxAXY/guxnP8BPfzgH2W0EBdh8tvnsY/cj0N92NgR8gsEKhLPP+e4TaLdVunpx1sJFHH3mGWwREbR97VWC+/Ytcb9ThU5e+eZnPlxzkJaNA/nz9b24NqY1xhiO5h3lX1v/xaI9iwiwBXBb15u4kyY0SfoCDqwExFqOuPcU6DUZwtqU+B6qftMg8aFBokojIqxIPkFGruPMB3igbzD4hEGwv41Aux9+NV2WvhoUJiWRMvNhXOnptHrqKcKn3lzqDLUth7P4w4JEktJOMbpbC569IYZ2Ta26ZAdPHeStLW/x9f6vaRzQmF/1+hW3to2l0a6vIXEupG0BDHS6zAqVHhPOa9VIVTdpkPjQIFENkTsri9Tf/o68n36iyeTJtP7TH/ELCipxX5fbw39XHeAf3+3GI8IjV17CPZd1wt9mTez8OfNn3tzyJssPL6dpUFPu63MfUy6ZQsDJg5A4DxI/h8x9YAuArtdYoXLJGPAPrs0uq2qmQeJDg0Q1VOJ2c+Kttzjx9jsE9exJ1OuvE9C29MWxjmQV8MySHXybdIxurRrzwuQYBnY4u+Tv1uNbeX3T66w7uo7WIa15sO+DTOg8AbuxwZHN1ihl+3zIPQYBjaHH9dZF+k6Xg61+3gDakGmQ+NAgUQ1dTlw8R554AuPnR+QrrxA6ckSZ+3+XdIw/L97OkexCpg1pxxNjuxPe6OxCWmvS1vD6ptdJPJFIx7COPNTvIa7peA1+xg88bjjwkxUqSV+AIxtCWljXUnpPsaoX68yvekGDxIcGiVJQdOAAKTMfxrFnDy0eeYRm9/waYy99lJDncPHq97uZlXCA8GB/nh7fg4n9os5caxER4g/H88bmN9iTtYduEd2Y2X8mo9qOOns9xlkIe76zQuXnZdZCXuEdzs78atm9NrquzpMGiQ8NEqUsnvx80v74J059+SX21q2JuOUWwm+egr1p01KP2XEkmz8s3M7Ww1mM6NKM526IIbrF2XIsbo+bZQeW8daWtzicc5i+LfryyIBHGNx68C9fqDAbdn1phcq+5SAeaNXbOvUVcyOEt6uhXqvzpUHiQ4NEqbNEhNz4eDJnzyZ/9RpMQABh48YRceutBPcuuVCE2yN8vPYgLy/7GYfbw0NXdOGBK6J/sVCY0+Nk0Z5F/Gvrv0jPT+fSNpfycP+H6d2i97kvmJsOOxZaoZKy3nqu/XDoMwV6ToRGpQebqj0aJD40SJQqmWPPHjI/+ojsxUuQ/HyC+/Yl4rbbCBtzDSYg4Jz9008V8uzSJJZuSyO6RQjPT4xheOdfLpzlcDv4bNdnvJ/4PicdJ4ltF8uM/jPoGtG15EZk7oPE+dbMrxO7wc8OXa6yTn11uxYCQmqi66oCNEh8aJAoVTZ3Tg7ZCxeS+dFHOA8ewta8ORE330z4LVPxb3lumfkfdx/nj4u2cygzn8kDonhqXI9zyszkOfOYkzSH/+74L3nOPMZFj+Ohvg/RLqyUU1gicDTx7MyvU6ng38iqStx7CnSOBZt/TXRflUKDxIcGiVIVIx4PeStXkjlnDnkrfgK7nbBrriHittsI7t/vFzc1FjrdvBGXzLsr9hESaOfJa7szZWC7c27YzHZkM2v7LD7e+TEuj4tJXSdxf5/7aRXSqvSGeDxwaLV35tciKDgJwU2h10QrVNpdCn41WbxcgQbJL2iQKFV5RQcPcvLjj8mavwBPbi5BPXtap72uG4df4NnRR/KxHJ5auJ11BzIZ3DGCv07qXeIKlMfzj/Ne4nvM3T0XP/y4pfst/Lr3r2kaVM71EFcR7I3zzvz6Cpz5ENYWet9ohUqrGJ1OXEM0SHxokCh1/jx5eWQvWULmRx9RtGcvtvBwwqdMIWLaLfhHRlr7eIR5G1N44eud5Ba6uG9UNDNjuxIccG4F4tTcVN7Z8g5f7PuCIFsQd/S6gzt63kHjgAosf+zIhZ+95Vn2/gAeF7To7p35dRM01SWEq5MGiQ8NEqWqTkTIX7uWzDlzyI2LB6DxlVcScdttNBoyGGMMGbkOXvhqF/M3pdCuaTDP3RDDFd1KXsp3X9Y+3tryFt8e/JYmgU24O+ZupnWfRrC9gmVV8jIgaaFVouXQauu5tkO8hSQnQWiL6uh2g6ZB4kODRKnqVZSSStann5A1dx7u7GwCu3Yl4tZbaTLhevwaNWL13gyeWpTIvuN5XNenDX8e35OWYSXX+UrKSOKNzW+wMnUlzYObc3+f+7mx6434V+bCetYh6wJ94jw4th2MDaKvsEJFlxA+bxokPjRIlKoZnsJCTn35JZlzPsKxcyd+YWGET55MxK3TkTaR/PvHfbwZv4dAmx+/HduNW4d2KLUs/sZjG3l90+tsSt9EVGgUD/Z9kPHR47FVdnXGY0mwfZ51+ivrENiD4JKxZwtJ6syvCtMg8aFBolTNEhEKNm0ic84ccr79DjweQkeNIuK220jv1pc/LtlBwp4M+rYL54VJMfSKbFLq6yQcSeD1Ta+zM3Mn0U2imdF/Ble1v6rUMvhlNMq62TFxLmxfAPknrJpf/aZD/zugeZdq6PnFTYPEhwaJUrXHeewYWZ99xsnPPsedkUFAx46E3zqdhOihPBt/kJP5Tn41vCOPXn0JIYEl1/oSEb4/9D1vbH6D/dn76dmsJzP7z2RE5IjKBwqA2wV7vofNs62L9eKGDiNhwB3Qc4KWuy9FnQgSY8xY4DXABrwvIn8rtv0u4O9AqvepN0Xkfe+2l4HrAD/gO+ARERFjzDKgDWAHfgIeEhF3We3QIFGq9nmKishZtozMOR9RuG0bfo0aETR+Ah9HDuXfB9xENgnimQm9uKZX61Jfw+1xs3TfUt7Z+g6puakMaDmAhwc8zMBWA8+/YTlHYcvHsOlDOLkfgppAn6lWqLQuoZxLA3bBg8QYYwN2A1cDKcB6YJqIJPnscxcwSERmFDt2OFbAjPI+tRJ4UkSWG2PCROSUsf5bMg+YKyKfltUWDRKlLqyCbds4+dFHnPrqa8TpxD1gMLNaDGZRUEeu7NWGv0zoRWR46aMCp9vJ/OT5/HvbvzlRcIKRUSOZ2X8mPZv1PP9GeTxwcKUVKElLrMrEkf2tQIm5SS/QUzeCZBjwjIiM8T5+EkBEXvTZ5y5KDpJhwJvASMAAK4DbRWSnzz7+wAJgjoh8VlZbNEiUqhtcJ06QNXcuJz/5FFd6OgXNW/Fp5FDio4dy/3X9uGt4R+y20u9YL3AV8MmuT5i1fRbZjmyu7nA1M/rNIDo8umoNy8+EbZ/Dpg8gPckqzdJrshUq7YY02Bse60KQ3ASMFZF7vI9vB4b6hoY3SF4EjmONXh4VkcPebf8L3IMVJG+KyFM+x30DDAG+xgoYPbWlVD0iTic5P/xA5pw5FGzYiNMewPdR/dk+dAwP33st/dqVve57TlEOHyZ9yIc7PqTQXcj10dfzYL8HiQotffXHijVMIHUTbPqvVUjSmWfd8DjgDuhzC4Q0q9rr1zN1IUimAGOKBckQEZnps08zIFdEHMaYB4CbRSTWGNMF69rKVO+u3wFPiMgKn2ODgI+Af4nIdyW8/33AfQDt27cfePDgwRrpp1Kqagp37iRzzkecXPIFfs4itjWPJnvMJKY+eitNQsu+CJ5ZmMl/Ev/Dp7s+xYOHm7rexH197qNFo2q4GdGRY5W63/gBpG6w1qPvPt4KlU6XN4haX3UhSMo9tVVsfxuQKSJNjDG/BYJE5Dnvtj8BhSLycrFj7gQGFz81VpyOSJSq+1wnT5L+2VyOfDCHkJPHyWgUDhMmM3Tm3fg3K3skcDTvKO9ue5eFyQux+9mZ3mM6d8fcTZPAkqcZV9qxHbBpNmz9BAqzrFUeB9wO/W6FsMjqeY86qC4EiR3rdNWVWLOy1gPTRWSHzz5tRCTN+/0krFHHpcaYqcC9wFisU1vLgFeBeKCxiKR5X/8j4CcRebOstmiQKFV/iNvN9nlfsv+9/6Nryi5cNn+CrhlD21/fRXBMrzKPPXTqEG9vfZuv9n1FiH8Id/a6k9t73k6IfzWtaeIshF1LrWsp+1eA8YOu18CAO62vttKXLq6PLniQeBsxDisAbMAsEfmrMeZZYIOILDHGvAhMAFxAJvCgiOzyjk7expq1JcAyEXnMGNMKWAoEel8zDuu6iqusdmiQKFX/uNwe5s7/iaMfzObyA+sJdhcR2LcvzcpYeOu05JPJvLn5TeIOxxERGME9ve9havepBNoCSz2m0jL2wuY5sOUjyD0Goa2tmx0H3A5Nq3jxv46oE0FSV2iQKFV/pWUX8OLnGzDffsnkg6toeeo4thbNibh5KuFTby5x4a3TEo8n8vrm11mTtobmwc0Z3W40I6JGMLT1UEIDQks9rlLcTkj+1ppGnPyttRZ9p1HWKKX7ePAvucZYfaBB4kODRKn674edx/jzokRa797C/Rkbabt7i7Xw1pgx1nrzxRbe8rUubR1zds5hbdpa8l352I2dfi37MSJqBCOjRtItotv53TFf3KkjsPkj2PyhVecrOMKa7TXgDmhVhXteLhANEh8aJEpdHPKLXLz2fTLvr9zPJa4sniraQYuV35a58JYvp9vJluNbWJm6klVHVrErcxcAzYObMzxyOCOjRjKszTDCg8qeflwujwf2/2hdS9m5FDxOaDvYCpRekyGwmkZDNUyDxIcGiVIXl51pp/jDwkQ2H8piVNsQng44gG3JPGvhrYgIa+GtW6aeWXirNMfzj7PqyCoSUhNYlbaKbEc2BkPv5r0ZETWCEVEjiGkWU/kKxL7yMmDbp9Y04hM/Q0AoxNxonfqKGlCnb3bUIPGhQaLUxcfjET5Zf4iXvt5FodPDA5dH86tGGeR++nGpC2+Vxe1xsyNjBwmpCaw8spLE44kIQlhAGMMjh1vBEjni/O9REYHD66xrKTsWWEsGt+wFA++0Stw3KmfJ4QtAg8SHBolSF6/jOQ6e/zKJxVuO0Kl5CM9PjGFwkKOEhbem0/jqq7GXc0/KaVmFWaxJW8PK1JUkHEngRMEJALpFdGN41HBGRo6kf8v+lVuA67TCU9aaKZs+hCObwRZoVSEecAd0vKzOjFI0SHxokCh18fsp+Th/XLSdAxn5TOwXydPje9LULr9YeAtjCO7Xj9DY0TSOjSUgOrpCF9lFhN0nd58Jlc3HNuMSF43sjRjSZggjI0cyImoEbRu3rXzD07ZZgbLtc3BkW1OH+3tvdmzc6jz+JqqPBokPDRKlGoZCp5u34/fwzo97Cfa38ftre3DL4HYYA46dO8mJiyc3Lo7CJKsIuX+H9jQeHUvo6NE0GjgAY6/YDYV5zjzWpa0j4UgCK1NXkpprrYTRMazjmVNgg1oPqvj68wDOAkhabIXKwQRrueBu1yL9b8fZ6UoKPeBweih0unG4zn51FHtc/OtDo7uUuipleTRIfGiQKNWw7EnP4amF21m7P5OBHSL466QYurc+WxbeefQoufHx5MTHk796DeJ04tekCaGjRtE4djQhl12GLfSXM6tEhCK35+yHtNODw+WmoMjNoZyDbD6+hm2Z60g+tQWXFGEz/rQNiqFdUH9a+fcjmNY4XILD5abQe+zprw6nh0Kfr62KDjPO+R3Xy3KamVOkSVM+d1/OXPcVpEjlrtHsfHYswQHnN1lAg8SHBolSDY+IMG9jCi98tZOcQhfThrSneWjgLz6wHU4P7rxcWidvo9PPG+m8dyshhbm4/Gzsbt2VjW1jWNemFykBTXC4PFTo49I4sTXajz1kN7bQ3dgC0wHwOMMxBd3xL+pOsKsHQfZGBNptBPn7EWj3I8jfds7XYD83PXNX0zd9MR1OrgbgWPNLOdxpCifbXU1AYFCJx53+PtDuV2ZZ/nK7okFylgaJUg1XZl4RL361k3mbUhABf5vx+QC3Eej9GuTvR7CfodPxfXTfu4XOyZuIyEgDICuyI8d7DyGr/6UUdepKYKB/qR/+xb9mOo6yMX0Nq9NWsSZtDXnOPOzGTt+WfRkZNZIRkSPo1rQbfqacD/ysw1Y5lk2z4VQKNGoGfadZF+hbdKuRvzsNEh8aJEoph8uN3c+vUtcLHPv2e0+BxVGwaTN4PNhbtiR09Ggax46m0aWXlnrzY0mcHidb07eScCSBhNQEdmZaa/U1C2p25trKsMhhRARFlP4iHjfsjbdudvz5K/C4oN2l3psdJ0JANRWoRIPkFzRIlFJV5Tp5ktwffyQ3Lp7clSuR/HxMo0aEjhhO6OhYQq+4HHvTyt0LcqLgBKuPrD5zp32WIwuDIaZ5zJk77WOax2D3K2USQG66Vdp+04eQsQcCw6D3TVaoRPavcp81SHxokCilqpPH4SB/3Tpy4uLIjYvHdeyYNbW4f38ax44mNDaWgE6dKlW/y+1xszNzpzXFODWBbSe24REPjQMaM6zNMEZGjWR45HBahZQwJVgEDq227p5PWgSuQmjdxwqUftPPe5SiQeJDg0QpVVNExJpa/EMcOfFxOJKs01UBHToQGhtL49jRBPfvX+GpxadlO7JZk7aGhFTrNFh6gXXRvmtE1zP3rfRv2Z8AW7Fy+gVZkDjXOvWVsRce/xmCwkp4h/JpkPjQIFFK1RZnWhq5y5eTExdP/hprarGtSRNCr7ic0NGxhIwciS20ciMEESE5K/lMqGxM34jL4yLYHszQ1kPP1AVr17id70GQnQLh7Up/4XJokPjQIFFKXQju3DzyEhLIjYsjd/ly3NnZGH9/Gg0dat1dP3o0/m3aVPp18535rDu6zqoLlrqSlNwUADqEdWBEpBUqg1sPrtwNkSXQIPGhQaKUutDE5aJgyxbr7voffqDo4EEAAnv0oPFo67pKUK+e57UuyqFTh86Ub1mXto5CdyEBfgEMbDWQl0a9VPYssDJokPjQIFFK1TXW1OI4cuLiKdjsnVrcqhWho6+gcWwsjYYOrdTU4jOv63aw6dgmElIT2J6xPzoHjgAACbtJREFUnVljZpV/j0opNEh8aJAopeoyV2YmuT+usE6BJST4TC0eQWisd2pxxPmNKqpCg8SHBolSqr7wOBzkr11rTS2OX25NLfbzOzu1eHQsgdGdaqUtdSJIjDFjgdcAG/C+iPyt2Pa7gL8Dqd6n3hSR973bXgauA/yA74BHgGBgLtAZcANfiMjvy2uHBolSqj4SEQqTksiNi//l1OKOHc9OLe7Xr9JTiyvqggeJMcYG7AauBlKA9cA0EUny2ecuYJCIzCh27HCsgBnlfWol8CSwDhgqIvHGmADgB+AFEfm6rLZokCilLgbOI0fIWb6c3Lh48tauBacTW3g4oZdfTmhsLCEjRlR6anFZKhokNRNjliHAHhHZ523Qp8ANQFKZR1kECAICAAP4A8dEJB+IBxCRImPMJuA8VpJRSqn6xz8ykqbTp9N0+nTcubnkrUwgN96aWpy9eLE1tfjSS72nwEbj37p1rbTr/OsLly8KOOzzOMX7XHE3GmO2GWPmGWPaAYjIaqzASPP++UZEdvoeZIwJB67HGpWcwxhznzFmgzFmw/Hjx6veG6WUqkNsoaGEjR1D5Esv0TVhJR1mf0jEbbdRdOggR//yLHuuGM2+yZNxZWTUeFtqMkhKmgxd/DzaF0BHEekDfA98AGCM6QL0wBptRAGxxpjTp7kwxtiBT4DXT494znkjkXdFZJCIDGrRonILwSilVH1i7HYaDR5Mqyd+R+dly4j+6kta/uZxAqKisFWykOT5qMlTWymA7735bYEjvjuIiG9Uvge85P1+ErBGRHIBjDFfA5cCK7zb3wWSReTVGmi3UkrVW8YYAqOjCYyOrrX3rMkRyXqgqzGmk/fC+C3AEt8djDG+tQEmAKdPXx0CLjfG2I0x/sDlp7cZY54HmgD/U4NtV0opVUE1NiIREZcxZgbwDdb031kissMY8yywQUSWAA8bYyYALiATuMt7+DwgFkjEOh22TES+MMa0BZ4CdgGbvKUEzkwZVkopVfv0hkSllFIlquj035o8taWUUqoB0CBRSilVJRokSimlqkSDRCmlVJVokCillKqSBjFryxhzHDh4noc3B05UY3PqA+1zw9DQ+tzQ+gtV73MHESm3NEiDCJKqMMZsqMj0t4uJ9rlhaGh9bmj9hdrrs57aUkopVSUaJEoppapEg6R8717oBlwA2ueGoaH1uaH1F2qpz3qNRCmlVJXoiEQppVSVaJB4GWPGGmN+NsbsMcb8voTtDxhjEo0xW4wxK40xPS9EO6tTeX322e8mY4wYY+r1jJcK/IzvMsYc9/6Mtxhj7rkQ7axOFfkZG2NuNsYkGWN2GGM+ru02VrcK/Jz/6fMz3m2MyboQ7axOFehze2NMvDFms3dF2nHV2gARafB/sMrc7wWisdaJ3wr0LLZPmM/3E7BK21/wttdkn737NcZaUGwNMOhCt7uGf8Z3YS1LcMHbW4t97gpsBiK8j1te6HbXdJ+L7T8Ta4mLC972Gv45vws86P2+J3CgOtugIxLLEGCPiOwTkSLgU+AG3x1E5JTPwxDOXTa4vim3z17PAS8DhbXZuBpQ0f5eTCrS53uBt0TkJICIpNdyG6tbZX/O07CW7a7PKtJnAcK83zeh2Gq1VaVBYokCDvs8TvE+9wvGmIeMMXuxPlgfrqW21ZRy+2yM6Q+0E5GltdmwGlKhnzFwo3foP88Y066E7fVJRfp8CXCJMSbBGLPGGDO21lpXMyr6c8YY0wHoBMTVQrtqUkX6/AxwmzEmBfgKayRWbTRILKaE584ZcYjIWyLSGXgCeLrGW1WzyuyzMcYP+CfweK21qGZV5Gf8BdBRRPoA3wMf1HiralZF+mzHOr11Bdb/zt83xoTXcLtqUoV+l71uAeaJiLsG21MbKtLnacB/RaQtMA6Y7f0drxYaJJYUwPd/n20pe+j3KTCxRltU88rrc2MgBlhujDkAXAosqccX3Mv9GYtIhog4vA/fAwbWUttqSkX+XacAi0XEKSL7gZ+xgqW+qszv8i3U/9NaULE+/xr4HEBEVgNBWHW4qoUGiWU90NUY08kYE4D1D2yJ7w7GGN9fruuA5FpsX00os88iki0izUWko4h0xLrYPkFE6uuaxRX5GbfxeTgB2FmL7asJ5fYZWASMBjDGNMc61bWvVltZvSrSZ4wx3YAIYHUtt68mVKTPh4ArAYwxPbCC5Hh1NcBeXS9Un4mIyxgzA/gGawbELBHZYYx5FtggIkuAGcaYqwAncBK488K1uOoq2OeLRgX7+7AxZgLgAjKxZnHVWxXs8zfANcaYJMAN/FZEMi5cq6umEv+upwGfincaU31WwT4/DrxnjHkU67TXXdXZd72zXSmlVJXoqS2llFJVokGilFKqSjRIlFJKVYkGiVJKqSrRIFFKKVUlGiRKVZEx5hljzG/qQDsOeO8FUapWaZAopZSqEg0SpUpgjAkxxnxpjNlqjNlujJnq+z9+Y8wgY8xyn0P6GmPijDHJxph7vfu0Mcas8K57sd0Yc5n3+XeMMRu863/8xec9DxhjXjDGrPZuH2CM+cYYs9cY84B3nyu8r7nQu4bIv0qqmWSMuc0Ys8773v82xthq8u9LNWwaJEqVbCxwRET6ikgMsKyc/ftglc4ZBvzJGBMJTAe+EZF+QF9gi3ffp0RkkPeYy40xfXxe57CIDAN+Av4L3IRV5+xZn32GYN2p3BvoDEz2bYi3BMZUYIT3vd3ArZXou1KVoiVSlCpZIvC/xpiXgKUi8pMxJRVZPWOxiBQABcaYeKwP+/XALGOMP7BIRE4Hyc3GmPuwfv/aYC00tM277XQJj0QgVERygBxjTKFPVd51IrIPwBjzCTASmOfTliuxCk6u97Y5GKjv64yoOkyDRKkSiMhuY8xArJLbLxpjvsWqwXV6FB9U/JBzX0JWGGNGYY1UZhtj/o410vgNMFhEThpj/lvstU5XH/b4fH/68enf13Peq9hjA3wgIk+W002lqoWe2lKqBN5TU/kiMgf4X2AAcICzpeVvLHbIDcaYIGNMM6y1PdZ7F05KF5H3gP94XyMMyAOyjTGtgGvPo3lDvJVe/bBOYa0stv0H4CZjTEtvX5p626JUjdARiVIl6w383Rjjwar4/CDWKaL/GGP+AKwttv864EugPfCciBwxxtwJ/NYY4wRygTtEZL8xZjOwA6tce8J5tG018DdvG1cAC303ikiSMeZp4Ftv2DiBh4CD5/FeSpVLq/8qVY8YY64AfiMi4y90W5Q6TU9tKaWUqhIdkSillKoSHZEopZSqEg0SpZRSVaJBopRSqko0SJRSSlWJBolSSqkq0SBRSilVJf8PAsD0A9Ns42sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch3_1.best_score_, gsearch3_1.best_params_))\n",
    "test_means = gsearch3_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch3_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch3_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch3_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch3_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二轮参数调整得到的n_estimators最优值（645），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58106, std: 0.00365, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58090, std: 0.00349, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58132, std: 0.00369, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58073, std: 0.00380, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58135, std: 0.00388, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58127, std: 0.00385, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       " -0.5807289533230032)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=610,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=1.5,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([396.09453249, 392.65251188, 388.26022778, 390.13807282,\n",
       "        376.94981999, 319.38518758]),\n",
       " 'std_fit_time': array([ 2.40290576,  3.22080772,  2.37260013,  2.91130978, 25.88850404,\n",
       "         1.44969896]),\n",
       " 'mean_score_time': array([3.78846645, 2.66804333, 3.84131203, 3.2198998 , 3.60466061,\n",
       "        1.80519533]),\n",
       " 'std_score_time': array([1.13102917, 1.31437371, 0.99871724, 1.43070957, 1.31400868,\n",
       "        0.29723637]),\n",
       " 'param_reg_alpha': masked_array(data=[1.5, 1.5, 1.5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'split0_test_score': array([-0.57462589, -0.57444728, -0.57477804, -0.57410597, -0.57442393,\n",
       "        -0.57476766]),\n",
       " 'split1_test_score': array([-0.5795916 , -0.58018629, -0.57968641, -0.57906729, -0.57975261,\n",
       "        -0.57909717]),\n",
       " 'split2_test_score': array([-0.58255854, -0.58220065, -0.58351572, -0.582229  , -0.58352395,\n",
       "        -0.58312165]),\n",
       " 'split3_test_score': array([-0.58376329, -0.58363849, -0.58431155, -0.5836505 , -0.58460769,\n",
       "        -0.58425134]),\n",
       " 'split4_test_score': array([-0.58474702, -0.58400926, -0.58430127, -0.58459317, -0.58444866,\n",
       "        -0.58512093]),\n",
       " 'mean_test_score': array([-0.58105704, -0.58089621, -0.58131842, -0.58072895, -0.58135118,\n",
       "        -0.58127151]),\n",
       " 'std_test_score': array([0.00365243, 0.00349303, 0.00369123, 0.00380375, 0.00388495,\n",
       "        0.00385093]),\n",
       " 'rank_test_score': array([3, 2, 5, 1, 6, 4], dtype=int32),\n",
       " 'split0_train_score': array([-0.48203507, -0.48333857, -0.48581547, -0.48406309, -0.48605623,\n",
       "        -0.48780459]),\n",
       " 'split1_train_score': array([-0.48046988, -0.4819405 , -0.48483281, -0.48270256, -0.48432037,\n",
       "        -0.48594744]),\n",
       " 'split2_train_score': array([-0.48047315, -0.48222449, -0.48433048, -0.48285682, -0.48412188,\n",
       "        -0.48608775]),\n",
       " 'split3_train_score': array([-0.48008201, -0.48126236, -0.48387669, -0.48248798, -0.48362735,\n",
       "        -0.48593602]),\n",
       " 'split4_train_score': array([-0.4807155 , -0.48168613, -0.48437357, -0.48302786, -0.48400375,\n",
       "        -0.48555732]),\n",
       " 'mean_train_score': array([-0.48075512, -0.48209041, -0.48464581, -0.48302766, -0.48442592,\n",
       "        -0.48626662]),\n",
       " 'std_train_score': array([0.00067138, 0.00069964, 0.00065851, 0.0005474 , 0.00084586,\n",
       "        0.00078888])}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid character in identifier (<ipython-input-68-06c83ef1b623>, line 8)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-68-06c83ef1b623>\"\u001b[0;36m, line \u001b[0;32m8\u001b[0m\n\u001b[0;31m    ​\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid character in identifier\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "​\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "​\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "​\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
